/*
# SPDX-FileCopyrightText: Copyright (c) 2020-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: LicenseRef-NvidiaProprietary
#
# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
# property and proprietary rights in and to this material, related
# documentation and any modifications thereto. Any use, reproduction,
# disclosure or distribution of this material and related documentation
# without an express license agreement from NVIDIA CORPORATION or
# its affiliates is strictly prohibited.
# ==============================================================================
*/
// -*- cuda -*-

#ifndef LIBCUDA_PTX_H

// Start prologue
// XXX: Ensure this prologue stays in sync with ../test/test_ptx_prologue.h
#define LIBCUDA_PTX_H

#include <cstdint>

// Forward-declare __half and __nv_bfloat16. The cuda_fp16.h and cuda_bf16.h are
// expensive to include. This is the user's responsibility (if they need the
// types).
struct __half;
struct __nv_bfloat16;

// Start of ./ptx_isa_target_macros.h
// -*- cuda -*-
#ifndef PTX_ISA_TARGET_MACROS_H
#define PTX_ISA_TARGET_MACROS_H

/*
 * Targeting macros
 *
 * Information from:
 * https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#release-notes
 */

#if       (defined(__libcuda_ptx_isa_override))
#  define __libcuda_ptx_isa __libcuda_ptx_isa_override
#elif     (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ > 13))
// Handle CTK 14
#  define __libcuda_ptx_isa 940ULL
#elif     (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 13 && __CUDACC_VER_MINOR__ >= 4)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 940ULL
#elif     (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 13 && __CUDACC_VER_MINOR__ >= 3)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 930ULL
#elif     (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 13 && __CUDACC_VER_MINOR__ >= 2)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 920ULL
#elif   (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 13 && __CUDACC_VER_MINOR__ >= 1)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 910ULL
#elif   (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 13 && __CUDACC_VER_MINOR__ >= 0)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 900ULL
#elif   (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 12 && __CUDACC_VER_MINOR__ >= 9)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 880ULL
#elif   (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 12 && __CUDACC_VER_MINOR__ >= 8)) || (!defined(__CUDACC_VER_MAJOR__))
// PTX ISA 8.7 - CUDA 12.8, driver r570
#  define __libcuda_ptx_isa 870ULL
#elif   (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 12 && __CUDACC_VER_MINOR__ >= 5)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 850ULL
#elif   (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 12 && __CUDACC_VER_MINOR__ >= 4)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 840ULL
// PTX ISA 8.3 is available from CUDA 12.3, driver r545
#elif   (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 12 && __CUDACC_VER_MINOR__ >= 3)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 830ULL
// PTX ISA 8.2 is available from CUDA 12.2, driver r535
#elif (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 12 && __CUDACC_VER_MINOR__ >= 2)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 820ULL
// PTX ISA 8.1 is available from CUDA 12.1, driver r530
#elif (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 12 && __CUDACC_VER_MINOR__ >= 1)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 810ULL
// PTX ISA 8.0 is available from CUDA 12.0, driver r525
#elif (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 12 && __CUDACC_VER_MINOR__ >= 0)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 800ULL
// PTX ISA 7.8 is available from CUDA 11.8, driver r520
#elif (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 11 && __CUDACC_VER_MINOR__ >= 8)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 780ULL
// PTX ISA 7.7 is available from CUDA 11.7, driver r515
#elif (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 11 && __CUDACC_VER_MINOR__ >= 7)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 770ULL
// PTX ISA 7.6 is available from CUDA 11.6, driver r510
#elif (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 11 && __CUDACC_VER_MINOR__ >= 6)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 760ULL
// PTX ISA 7.5 is available from CUDA 11.5, driver r495
#elif (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 11 && __CUDACC_VER_MINOR__ >= 5)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 750ULL
// PTX ISA 7.4 is available from CUDA 11.4, driver r470
#elif (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 11 && __CUDACC_VER_MINOR__ >= 4)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 740ULL
// PTX ISA 7.3 is available from CUDA 11.3, driver r465
#elif (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 11 && __CUDACC_VER_MINOR__ >= 3)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 730ULL
// PTX ISA 7.2 is available from CUDA 11.2, driver r460
#elif (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 11 && __CUDACC_VER_MINOR__ >= 2)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 720ULL
// PTX ISA 7.1 is available from CUDA 11.1, driver r455
#elif (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 11 && __CUDACC_VER_MINOR__ >= 1)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 710ULL
// PTX ISA 7.0 is available from CUDA 11.0, driver r445
#elif (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 11 && __CUDACC_VER_MINOR__ >= 0)) || (!defined(__CUDACC_VER_MAJOR__))
#  define __libcuda_ptx_isa 700ULL
// Fallback case. Define the ISA version to be zero. This ensures that the macro is always defined.
#else
#  define __libcuda_ptx_isa 0ULL
#endif

#endif // PTX_ISA_TARGET_MACROS_H

// End of ./ptx_isa_target_macros.h
// Start of ./ptx_dot_variants.h
// -*- C++ -*-
#ifndef _LIBCUDA_PTX_DOT_VARIANTS_H_
#define _LIBCUDA_PTX_DOT_VARIANTS_H_

/*
 * Public integral constant types and values for ".variant"s:
 *
 * - .sem:     acquire, release, ..
 * - .space:   global, shared, constant, ..
 * - .scope:   cta, cluster, gpu, ..
 * - .op:      add, min, cas, ..
 *
 * For each .variant, the code below defines:
 * - An enum `dot_variant` with each possible value
 * - A type template `variant_t<dot_variant>`
 * - Types `variant_A_t`, ..., `variant_Z_t`
 * - Constexpr values `variant_A` of type `variant_A_t`
 *
 * These types enable specifying fine-grained overloads of a PTX binding. If a
 * binding can handle multiple variants, then it is defined as:
 *
 * template <dot_variant var>
 * [...] void ptx_binding(variant_t<var> __v) { ... }
 *
 * If it only handles a single variant, then it is defined as:
 *
 * [...] void ptx_binding(variant_A __v) { ... }
 *
 * If two variants have different behaviors or return types (see .space
 * overloads of mbarrier.arrive.expect_tx for an example), then these can be
 * provided as separate overloads of the same function:
 *
 * [...] void ptx_binding(variant_A __v) { ... }
 * [...] int ptx_binding(variant_B __v) { ... }
 *
 */

namespace cuda_ptx {

template<class T, T v>
struct __integral_constant
{
    static constexpr T value = v;
    using value_type = T;
    using type = __integral_constant; // using injected-class-name
    constexpr __host__ __device__  operator value_type() const noexcept { return value; }
    constexpr __host__ __device__ value_type operator()() const noexcept { return value; } // since c++14
};

// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#operation-types
enum class dot_sem
{
  acq_rel,
  acquire,
  relaxed,
  release,
  sc,
  weak
};

// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#state-spaces
enum class dot_space
{
  global,
  cluster, // The PTX spelling is shared::cluster
  shared,  // The PTX spelling is shared::cta

  // The following state spaces are unlikely to be used in cuda::ptx in the near
  // future, so they are not exposed:

  // reg,
  // sreg,
  // const_mem, // Using const_mem as `const` is reserved in C++.
  // local,
  // param,
  // tex // deprecated
};

// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#scope
enum class dot_scope
{
  cta,
  cluster,
  gpu,
  sys
};

enum class dot_op
{
  add,
  dec,
  inc,
  max,
  min,
  and_op, // Using and_op, as `and, or, xor` are reserved in C++.
  or_op,
  xor_op,
  cas,
  exch
};

enum class dot_cta_group
{
   cta_group_1,
   cta_group_2
};

enum class dot_kind
{
 f16,
 f8f6f4,
 i8,
 mxf4,
 mxf4nvf4,
 mxf8f6f4,
 tf32
};


template <dot_sem __sem>
using sem_t         = __integral_constant<dot_sem, __sem>;
using sem_acq_rel_t = sem_t<dot_sem::acq_rel>;
using sem_acquire_t = sem_t<dot_sem::acquire>;
using sem_relaxed_t = sem_t<dot_sem::relaxed>;
using sem_release_t = sem_t<dot_sem::release>;
using sem_sc_t      = sem_t<dot_sem::sc>;
using sem_weak_t    = sem_t<dot_sem::weak>;

[[maybe_unused]] static constexpr sem_acq_rel_t sem_acq_rel{};
[[maybe_unused]] static constexpr sem_acquire_t sem_acquire{};
[[maybe_unused]] static constexpr sem_relaxed_t sem_relaxed{};
[[maybe_unused]] static constexpr sem_release_t sem_release{};
[[maybe_unused]] static constexpr sem_sc_t sem_sc{};
[[maybe_unused]] static constexpr sem_weak_t sem_weak{};

template <dot_space __spc>
using space_t         = __integral_constant<dot_space, __spc>;
using space_global_t  = space_t<dot_space::global>;
using space_shared_t  = space_t<dot_space::shared>;
using space_cluster_t = space_t<dot_space::cluster>;

[[maybe_unused]] static constexpr space_global_t space_global{};
[[maybe_unused]] static constexpr space_shared_t space_shared{};
[[maybe_unused]] static constexpr space_cluster_t space_cluster{};

template <dot_scope __scope>
using scope_t         = __integral_constant<dot_scope, __scope>;
using scope_cluster_t = scope_t<dot_scope::cluster>;
using scope_cta_t     = scope_t<dot_scope::cta>;
using scope_gpu_t     = scope_t<dot_scope::gpu>;
using scope_sys_t     = scope_t<dot_scope::sys>;

[[maybe_unused]] static constexpr scope_cluster_t scope_cluster{};
[[maybe_unused]] static constexpr scope_cta_t scope_cta{};
[[maybe_unused]] static constexpr scope_gpu_t scope_gpu{};
[[maybe_unused]] static constexpr scope_sys_t scope_sys{};

template <dot_op __op>
using op_t        = __integral_constant<dot_op, __op>;
using op_add_t    = op_t<dot_op::add>;
using op_dec_t    = op_t<dot_op::dec>;
using op_inc_t    = op_t<dot_op::inc>;
using op_max_t    = op_t<dot_op::max>;
using op_min_t    = op_t<dot_op::min>;
using op_and_op_t = op_t<dot_op::and_op>;
using op_or_op_t  = op_t<dot_op::or_op>;
using op_xor_op_t = op_t<dot_op::xor_op>;
using op_cas_t    = op_t<dot_op::cas>;
using op_exch_t   = op_t<dot_op::exch>;

[[maybe_unused]] static constexpr op_add_t op_add{};
[[maybe_unused]] static constexpr op_dec_t op_dec{};
[[maybe_unused]] static constexpr op_inc_t op_inc{};
[[maybe_unused]] static constexpr op_max_t op_max{};
[[maybe_unused]] static constexpr op_min_t op_min{};
[[maybe_unused]] static constexpr op_and_op_t op_and_op{};
[[maybe_unused]] static constexpr op_or_op_t op_or_op{};
[[maybe_unused]] static constexpr op_xor_op_t op_xor_op{};
[[maybe_unused]] static constexpr op_cas_t op_cas{};
[[maybe_unused]] static constexpr op_exch_t op_exch{};

template <dot_cta_group __cta_group>
using cta_group_t         = __integral_constant<dot_cta_group, __cta_group>;
using cta_group_1_t       = cta_group_t<dot_cta_group::cta_group_1>;
using cta_group_2_t       = cta_group_t<dot_cta_group::cta_group_2>;

[[maybe_unused]] static constexpr cta_group_1_t cta_group_1{};
[[maybe_unused]] static constexpr cta_group_2_t cta_group_2{};

template <dot_kind __kind>
using kind_t           = __integral_constant<dot_kind, __kind>;
using kind_f16_t       = kind_t<dot_kind::f16>;
using kind_f8f6f4_t    = kind_t<dot_kind::f8f6f4>;
using kind_i8_t        = kind_t<dot_kind::i8>;
using kind_mxf4_t      = kind_t<dot_kind::mxf4>;
using kind_mxf4nvf4_t  = kind_t<dot_kind::mxf4nvf4>;
using kind_mxf8f6f4_t  = kind_t<dot_kind::mxf8f6f4>;
using kind_tf32_t      = kind_t<dot_kind::tf32>;

[[maybe_unused]] static constexpr kind_f16_t      kind_f16{};
[[maybe_unused]] static constexpr kind_f8f6f4_t   kind_f8f6f4{};
[[maybe_unused]] static constexpr kind_i8_t       kind_i8{};
[[maybe_unused]] static constexpr kind_mxf4_t     kind_mxf4{};
[[maybe_unused]] static constexpr kind_mxf4nvf4_t kind_mxf4nvf4{};
[[maybe_unused]] static constexpr kind_mxf8f6f4_t kind_mxf8f6f4{};
[[maybe_unused]] static constexpr kind_tf32_t     kind_tf32{};


template <int n>
using n32_t = __integral_constant<int, n>;



} // namespace cuda_ptx

#endif // _LIBCUDA_PTX_DOT_VARIANTS_H_

// End of ./ptx_dot_variants.h
// Start of ./ptx_helper_functions.h
// -*- cuda -*-
#ifndef _LIBCUDA_PTX_HELPER_FUNCTIONS_H_
#define _LIBCUDA_PTX_HELPER_FUNCTIONS_H_

#include <cstdint>

#if defined(__CUDACC__) || defined(_NVHPC_CUDA) || defined(__CUDACC_RTC__)
#define _LIBCUDA_PTX_CUDACC_MAJOR() __CUDACC_VER_MAJOR__
#elif defined(__CUDA__) && defined(__clang__)
#define _LIBCUDA_PTX_CUDACC_MAJOR() (CUDA_VERSION / 1000)
#endif // ^^^ has cuda compiler ^^^

#if !defined(_LIBCUDA_PTX_ARCH_SPECIFIC)
#  if defined(__CUDA_ARCH_SPECIFIC__)
#    define  _LIBCUDA_PTX_ARCH_SPECIFIC() __CUDA_ARCH_SPECIFIC__
#  else
#    if defined(__CUDA_ARCH_FEAT_SM90_ALL)
#      define _LIBCUDA_PTX_ARCH_SPECIFIC() 900
#    elif defined(__CUDA_ARCH_FEAT_SM100_ALL)
#      define _LIBCUDA_PTX_ARCH_SPECIFIC() 1000
#    elif defined(__CUDA_ARCH_FEAT_SM103_ALL)
#      define _LIBCUDA_PTX_ARCH_SPECIFIC() 1030
#    elif defined(__CUDA_ARCH_FEAT_SM120_ALL)
#      define _LIBCUDA_PTX_ARCH_SPECIFIC() 1200
#    else
#      define _LIBCUDA_PTX_ARCH_SPECIFIC() 0
#    endif
#  endif // ^^^ !defined(__CUDA_ARCH_SPECIFIC__)
#endif // ^^^ !defined(_LIBCUDA_PTX_ARCH_SPECIFIC)

#if !defined(__CUDA_HAS_ARCH_FAMILY_SPECIFIC)

#define __CUDA_HAS_ARCH_FAMILY_SPECIFIC(N) false

#endif // !defined(__CUDA_HAS_ARCH_FAMILY_SPECIFIC)

namespace cuda_ptx {

#if _LIBCUDA_PTX_CUDACC_MAJOR() < 13
struct alignas(32) longlong4_32a
{
  long long x, y, z, w;
};
struct alignas(32) ulonglong4_32a
{
  unsigned long long x, y, z, w;
};
struct alignas(32) double4_32a
{
  double x, y, z, w;
};
#else
using ::double4_32a;
using ::longlong4_32a;
using ::ulonglong4_32a;
#endif // _LIBCUDA_PTX_CUDACC_MAJOR() < 13

/*************************************************************
 *
 * Conversion from generic pointer -> state space "pointer"
 *
 **************************************************************/
static inline __device__ std::uint32_t __as_ptr_smem(const void* __ptr)
{
  // Consider adding debug asserts here.
  return static_cast<std::uint32_t>(__cvta_generic_to_shared(__ptr));
}

static inline __device__ std::uint32_t __as_ptr_remote_dsmem(const void* __ptr)
{
  // No difference in implementation to __as_ptr_smem.
  // Consider adding debug asserts here.
  return __as_ptr_smem(__ptr);
}

static inline __device__ std::uint32_t __as_ptr_dsmem(const void* __ptr)
{
  // No difference in implementation to __as_ptr_smem.
  return __as_ptr_smem(__ptr);
}

static inline __device__ std::uint64_t __as_ptr_gmem(const void* __ptr)
{
  // Consider adding debug asserts here.
  return static_cast<std::uint64_t>(__cvta_generic_to_global(__ptr));
}

/*************************************************************
 *
 * Conversion from state space "pointer" -> generic pointer
 *
 **************************************************************/
template <typename _Tp>
static inline __device__ _Tp* __from_ptr_smem(size_t __ptr)
{
  // Consider adding debug asserts here.
  return reinterpret_cast<_Tp*>(__cvta_shared_to_generic(__ptr));
}

template <typename _Tp>
static inline __device__ _Tp* __from_ptr_dsmem(size_t __ptr)
{
  // Consider adding debug asserts here.
  return __from_ptr_smem<_Tp>(__ptr);
}

template <typename _Tp>
static inline __device__ _Tp* __from_ptr_remote_dsmem(size_t __ptr)
{
  // Consider adding debug asserts here.
  return __from_ptr_smem<_Tp>(__ptr);
}

template <typename _Tp>
static inline __device__ _Tp* __from_ptr_gmem(size_t __ptr)
{
  // Consider adding debug asserts here.
  return reinterpret_cast<_Tp*>(__cvta_global_to_generic(__ptr));
}

/*************************************************************
 *
 * Conversion to and from b8 type
 *
 **************************************************************/

template <typename _B8>
__device__ static inline uint32_t __b8_as_u32(_B8 __val) {
  static_assert(sizeof(_B8) == 1, "");
  uint32_t __u32 = 0;
  memcpy(&__u32, &__val, 1);
  return __u32;
}

template <typename _B8>
__device__ static inline _B8 __u32_as_b8(uint32_t __u32) {
  static_assert(sizeof(_B8) == 1, "");
  _B8 b8;
  memcpy(&b8, &__u32, 1);
  return b8;
}

} // namespace cuda_ptx

#endif // _LIBCUDA_PTX_HELPER_FUNCTIONS_H_

// End of ./ptx_helper_functions.h
// End prologue

// Start of ./instructions/barrier_cluster.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_BARRIER_CLUSTER_H
#define INSTRUCTIONS_BARRIER_CLUSTER_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/barrier_cluster.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {


/*
// barrier.cluster.arrive; // PTX ISA 78, SM_90
// Marked volatile and as clobbering memory
template <typename = void>
__device__ static inline void barrier_cluster_arrive();
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_barrier_cluster_arrive_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void barrier_cluster_arrive()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "barrier.cluster.arrive;"
      :
      :
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_barrier_cluster_arrive_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 780


/*
// barrier.cluster.wait; // PTX ISA 78, SM_90
// Marked volatile and as clobbering memory
template <typename = void>
__device__ static inline void barrier_cluster_wait();
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_barrier_cluster_wait_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void barrier_cluster_wait()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "barrier.cluster.wait;"
      :
      :
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_barrier_cluster_wait_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 780


/*
// barrier.cluster.arrive.sem; // PTX ISA 80, SM_90
// .sem       = { .release }
// Marked volatile and as clobbering memory
template <typename = void>
__device__ static inline void barrier_cluster_arrive(
  cuda_ptx::sem_release_t);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_barrier_cluster_arrive_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void barrier_cluster_arrive(
  cuda_ptx::sem_release_t)
{
  // sem == sem_release (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "barrier.cluster.arrive.release;"
      :
      :
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_barrier_cluster_arrive_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800


/*
// barrier.cluster.arrive.sem; // PTX ISA 80, SM_90
// .sem       = { .relaxed }
// Marked volatile
template <typename = void>
__device__ static inline void barrier_cluster_arrive(
  cuda_ptx::sem_relaxed_t);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_barrier_cluster_arrive_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void barrier_cluster_arrive(
  cuda_ptx::sem_relaxed_t)
{
  // sem == sem_relaxed (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "barrier.cluster.arrive.relaxed;"
      :
      :
      :
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_barrier_cluster_arrive_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800


/*
// barrier.cluster.wait.sem; // PTX ISA 80, SM_90
// .sem       = { .acquire }
// Marked volatile and as clobbering memory
template <typename = void>
__device__ static inline void barrier_cluster_wait(
  cuda_ptx::sem_acquire_t);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_barrier_cluster_wait_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void barrier_cluster_wait(
  cuda_ptx::sem_acquire_t)
{
  // sem == sem_acquire (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "barrier.cluster.wait.acquire;"
      :
      :
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_barrier_cluster_wait_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

} // namespace cuda_ptx

#endif // INSTRUCTIONS_BARRIER_CLUSTER_H

// End of ./instructions/barrier_cluster.h
// Start of ./instructions/bfind.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_BFIND_H
#define INSTRUCTIONS_BFIND_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/bfind.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// bfind.u32 dest, a_reg; // PTX ISA 20, SM_50
template <typename = void>
__device__ static inline std::uint32_t bfind(
  std::uint32_t a_reg);
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_bfind_is_not_supported_before_SM_50__();
template <typename = void>
__device__ static inline std::uint32_t bfind(
  std::uint32_t a_reg)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "bfind.u32 %0, %1;"
      : "=r"(dest)
      : "r"(a_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_bfind_is_not_supported_before_SM_50__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// bfind.shiftamt.u32 dest, a_reg; // PTX ISA 20, SM_50
template <typename = void>
__device__ static inline std::uint32_t bfind_shiftamt(
  std::uint32_t a_reg);
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_bfind_shiftamt_is_not_supported_before_SM_50__();
template <typename = void>
__device__ static inline std::uint32_t bfind_shiftamt(
  std::uint32_t a_reg)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "bfind.shiftamt.u32 %0, %1;"
      : "=r"(dest)
      : "r"(a_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_bfind_shiftamt_is_not_supported_before_SM_50__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// bfind.u64 dest, a_reg; // PTX ISA 20, SM_50
template <typename = void>
__device__ static inline std::uint32_t bfind(
  std::uint64_t a_reg);
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_bfind_is_not_supported_before_SM_50__();
template <typename = void>
__device__ static inline std::uint32_t bfind(
  std::uint64_t a_reg)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "bfind.u64 %0, %1;"
      : "=r"(dest)
      : "l"(a_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_bfind_is_not_supported_before_SM_50__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// bfind.shiftamt.u64 dest, a_reg; // PTX ISA 20, SM_50
template <typename = void>
__device__ static inline std::uint32_t bfind_shiftamt(
  std::uint64_t a_reg);
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_bfind_shiftamt_is_not_supported_before_SM_50__();
template <typename = void>
__device__ static inline std::uint32_t bfind_shiftamt(
  std::uint64_t a_reg)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "bfind.shiftamt.u64 %0, %1;"
      : "=r"(dest)
      : "l"(a_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_bfind_shiftamt_is_not_supported_before_SM_50__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// bfind.s32 dest, a_reg; // PTX ISA 20, SM_50
template <typename = void>
__device__ static inline std::uint32_t bfind(
  std::int32_t a_reg);
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_bfind_is_not_supported_before_SM_50__();
template <typename = void>
__device__ static inline std::uint32_t bfind(
  std::int32_t a_reg)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "bfind.s32 %0, %1;"
      : "=r"(dest)
      : "r"(a_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_bfind_is_not_supported_before_SM_50__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// bfind.shiftamt.s32 dest, a_reg; // PTX ISA 20, SM_50
template <typename = void>
__device__ static inline std::uint32_t bfind_shiftamt(
  std::int32_t a_reg);
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_bfind_shiftamt_is_not_supported_before_SM_50__();
template <typename = void>
__device__ static inline std::uint32_t bfind_shiftamt(
  std::int32_t a_reg)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "bfind.shiftamt.s32 %0, %1;"
      : "=r"(dest)
      : "r"(a_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_bfind_shiftamt_is_not_supported_before_SM_50__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// bfind.s64 dest, a_reg; // PTX ISA 20, SM_50
template <typename = void>
__device__ static inline std::uint32_t bfind(
  std::int64_t a_reg);
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_bfind_is_not_supported_before_SM_50__();
template <typename = void>
__device__ static inline std::uint32_t bfind(
  std::int64_t a_reg)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "bfind.s64 %0, %1;"
      : "=r"(dest)
      : "l"(a_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_bfind_is_not_supported_before_SM_50__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// bfind.shiftamt.s64 dest, a_reg; // PTX ISA 20, SM_50
template <typename = void>
__device__ static inline std::uint32_t bfind_shiftamt(
  std::int64_t a_reg);
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_bfind_shiftamt_is_not_supported_before_SM_50__();
template <typename = void>
__device__ static inline std::uint32_t bfind_shiftamt(
  std::int64_t a_reg)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "bfind.shiftamt.s64 %0, %1;"
      : "=r"(dest)
      : "l"(a_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_bfind_shiftamt_is_not_supported_before_SM_50__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

} // namespace cuda_ptx

#endif // INSTRUCTIONS_BFIND_H

// End of ./instructions/bfind.h
// Start of ./instructions/bmsk.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_BMSK_H
#define INSTRUCTIONS_BMSK_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/bmsk.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// bmsk.clamp.b32 dest, a_reg, b_reg; // PTX ISA 76, SM_70
template <typename = void>
__device__ static inline std::uint32_t bmsk_clamp(
  std::uint32_t a_reg,
  std::uint32_t b_reg);
*/
#if __libcuda_ptx_isa >= 760
extern "C" __device__ void __cuda_ptx_bmsk_clamp_is_not_supported_before_SM_70__();
template <typename = void>
__device__ static inline std::uint32_t bmsk_clamp(
  std::uint32_t a_reg,
  std::uint32_t b_reg)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint32_t dest;
    asm (
      "bmsk.clamp.b32 %0, %1, %2;"
      : "=r"(dest)
      : "r"(a_reg),
        "r"(b_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_bmsk_clamp_is_not_supported_before_SM_70__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 760

/*
// bmsk.wrap.b32 dest, a_reg, b_reg; // PTX ISA 76, SM_70
template <typename = void>
__device__ static inline std::uint32_t bmsk_wrap(
  std::uint32_t a_reg,
  std::uint32_t b_reg);
*/
#if __libcuda_ptx_isa >= 760
extern "C" __device__ void __cuda_ptx_bmsk_wrap_is_not_supported_before_SM_70__();
template <typename = void>
__device__ static inline std::uint32_t bmsk_wrap(
  std::uint32_t a_reg,
  std::uint32_t b_reg)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint32_t dest;
    asm (
      "bmsk.wrap.b32 %0, %1, %2;"
      : "=r"(dest)
      : "r"(a_reg),
        "r"(b_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_bmsk_wrap_is_not_supported_before_SM_70__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 760

} // namespace cuda_ptx

#endif // INSTRUCTIONS_BMSK_H

// End of ./instructions/bmsk.h
// Start of ./instructions/clusterlaunchcontrol.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_CLUSTERLAUNCHCONTROL_H
#define INSTRUCTIONS_CLUSTERLAUNCHCONTROL_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/clusterlaunchcontrol.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [addr], [smem_bar]; // PTX ISA 86, SM_100
template <typename = void>
__device__ static inline void clusterlaunchcontrol_try_cancel(
  void* addr,
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_clusterlaunchcontrol_try_cancel_is_not_supported_before_SM_100__();
template <typename = void>
__device__ static inline void clusterlaunchcontrol_try_cancel(
  void* addr,
  std::uint64_t* smem_bar)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    asm (
      "clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [%0], [%1];"
      :
      : "r"(__as_ptr_smem(addr)),
        "r"(__as_ptr_smem(smem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_clusterlaunchcontrol_try_cancel_is_not_supported_before_SM_100__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.multicast::cluster::all.b128 [addr], [smem_bar]; // PTX ISA 86, SM_100a, SM_110a
template <typename = void>
__device__ static inline void clusterlaunchcontrol_try_cancel_multicast(
  void* addr,
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_clusterlaunchcontrol_try_cancel_multicast_is_only_supported_on_SM_100a_110a__();
template <typename = void>
__device__ static inline void clusterlaunchcontrol_try_cancel_multicast(
  void* addr,
  std::uint64_t* smem_bar)
{
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    asm (
      "clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.multicast::cluster::all.b128 [%0], [%1];"
      :
      : "r"(__as_ptr_smem(addr)),
        "r"(__as_ptr_smem(smem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_clusterlaunchcontrol_try_cancel_multicast_is_only_supported_on_SM_100a_110a__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// clusterlaunchcontrol.query_cancel.is_canceled.pred.b128 pred_is_canceled, try_cancel_response; // PTX ISA 86, SM_100
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline bool clusterlaunchcontrol_query_cancel_is_canceled(
  B128 try_cancel_response);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_clusterlaunchcontrol_query_cancel_is_canceled_is_not_supported_before_SM_100__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline bool clusterlaunchcontrol_query_cancel_is_canceled(
  B128 try_cancel_response)
{
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    std::uint32_t pred_is_canceled;
    asm (
      "{\n\t"
      ".reg .pred P_OUT; \n\t"
      ".reg .b128 B128_try_cancel_response; \n\t"
      "mov.b128 B128_try_cancel_response, {%1, %2}; \n\t"
      "clusterlaunchcontrol.query_cancel.is_canceled.pred.b128 P_OUT, B128_try_cancel_response; \n\t"
      "selp.b32 %0, 1, 0, P_OUT; \n"
      "}"
      : "=r"(pred_is_canceled)
      : "l"((*reinterpret_cast<longlong2*>(&try_cancel_response)).x),
        "l"((*reinterpret_cast<longlong2*>(&try_cancel_response)).y)
      :
    );
    return static_cast<bool>(pred_is_canceled);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_clusterlaunchcontrol_query_cancel_is_canceled_is_not_supported_before_SM_100__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// clusterlaunchcontrol.query_cancel.get_first_ctaid::x.b32.b128 ret_dim, try_cancel_response; // PTX ISA 86, SM_100
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B32 clusterlaunchcontrol_query_cancel_get_first_ctaid_x(
  B128 try_cancel_response);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_x_is_not_supported_before_SM_100__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B32 clusterlaunchcontrol_query_cancel_get_first_ctaid_x(
  B128 try_cancel_response)
{
  static_assert(sizeof(B32) == 4, "");
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    std::uint32_t ret_dim;
    asm (
      "{\n\t"
      ".reg .b128 B128_try_cancel_response; \n\t"
      "mov.b128 B128_try_cancel_response, {%1, %2}; \n\t"
      "clusterlaunchcontrol.query_cancel.get_first_ctaid::x.b32.b128 %0, B128_try_cancel_response; \n"
      "}"
      : "=r"(ret_dim)
      : "l"((*reinterpret_cast<longlong2*>(&try_cancel_response)).x),
        "l"((*reinterpret_cast<longlong2*>(&try_cancel_response)).y)
      :
    );
    return *reinterpret_cast<B32*>(&ret_dim);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_x_is_not_supported_before_SM_100__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// clusterlaunchcontrol.query_cancel.get_first_ctaid::y.b32.b128 ret_dim, try_cancel_response; // PTX ISA 86, SM_100
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B32 clusterlaunchcontrol_query_cancel_get_first_ctaid_y(
  B128 try_cancel_response);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_y_is_not_supported_before_SM_100__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B32 clusterlaunchcontrol_query_cancel_get_first_ctaid_y(
  B128 try_cancel_response)
{
  static_assert(sizeof(B32) == 4, "");
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    std::uint32_t ret_dim;
    asm (
      "{\n\t"
      ".reg .b128 B128_try_cancel_response; \n\t"
      "mov.b128 B128_try_cancel_response, {%1, %2}; \n\t"
      "clusterlaunchcontrol.query_cancel.get_first_ctaid::y.b32.b128 %0, B128_try_cancel_response; \n"
      "}"
      : "=r"(ret_dim)
      : "l"((*reinterpret_cast<longlong2*>(&try_cancel_response)).x),
        "l"((*reinterpret_cast<longlong2*>(&try_cancel_response)).y)
      :
    );
    return *reinterpret_cast<B32*>(&ret_dim);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_y_is_not_supported_before_SM_100__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// clusterlaunchcontrol.query_cancel.get_first_ctaid::z.b32.b128 ret_dim, try_cancel_response; // PTX ISA 86, SM_100
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B32 clusterlaunchcontrol_query_cancel_get_first_ctaid_z(
  B128 try_cancel_response);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_z_is_not_supported_before_SM_100__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B32 clusterlaunchcontrol_query_cancel_get_first_ctaid_z(
  B128 try_cancel_response)
{
  static_assert(sizeof(B32) == 4, "");
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    std::uint32_t ret_dim;
    asm (
      "{\n\t"
      ".reg .b128 B128_try_cancel_response; \n\t"
      "mov.b128 B128_try_cancel_response, {%1, %2}; \n\t"
      "clusterlaunchcontrol.query_cancel.get_first_ctaid::z.b32.b128 %0, B128_try_cancel_response; \n"
      "}"
      : "=r"(ret_dim)
      : "l"((*reinterpret_cast<longlong2*>(&try_cancel_response)).x),
        "l"((*reinterpret_cast<longlong2*>(&try_cancel_response)).y)
      :
    );
    return *reinterpret_cast<B32*>(&ret_dim);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_z_is_not_supported_before_SM_100__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 block_dim, try_cancel_response; // PTX ISA 86, SM_100
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline void clusterlaunchcontrol_query_cancel_get_first_ctaid(
  B32 (&block_dim)[4],
  B128 try_cancel_response);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_is_not_supported_before_SM_100__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline void clusterlaunchcontrol_query_cancel_get_first_ctaid(
  B32 (&block_dim)[4],
  B128 try_cancel_response)
{
  static_assert(sizeof(B32) == 4, "");
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    asm (
      "{\n\t"
      ".reg .b128 B128_try_cancel_response; \n\t"
      "mov.b128 B128_try_cancel_response, {%4, %5}; \n\t"
      "clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 {%0, %1, %2, %3}, B128_try_cancel_response; \n"
      "}"
      : "=r"(block_dim[0]),
        "=r"(block_dim[1]),
        "=r"(block_dim[2]),
        "=r"(block_dim[3])
      : "l"((*reinterpret_cast<longlong2*>(&try_cancel_response)).x),
        "l"((*reinterpret_cast<longlong2*>(&try_cancel_response)).y)
      :
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_clusterlaunchcontrol_query_cancel_get_first_ctaid_is_not_supported_before_SM_100__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

} // namespace cuda_ptx

#endif // INSTRUCTIONS_CLUSTERLAUNCHCONTROL_H

// End of ./instructions/clusterlaunchcontrol.h
// Start of ./instructions/cp_async_bulk.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_CP_ASYNC_BULK_H
#define INSTRUCTIONS_CP_ASYNC_BULK_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/cp_async_bulk.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// cp.async.bulk.dst.src.mbarrier::complete_tx::bytes [dstMem], [srcMem], size, [smem_bar]; // PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* srcMem,
  const std::uint32_t& size,
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* srcMem,
  const std::uint32_t& size,
  std::uint64_t* smem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(__as_ptr_gmem(srcMem)),
        "r"(size),
        "r"(__as_ptr_smem(smem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.dst.src.mbarrier::complete_tx::bytes [dstMem], [srcMem], size, [smem_bar]; // PTX ISA 86, SM_90
// .dst       = { .shared::cta }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* srcMem,
  const std::uint32_t& size,
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* srcMem,
  const std::uint32_t& size,
  std::uint64_t* smem_bar)
{
  // space == space_shared (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(__as_ptr_gmem(srcMem)),
        "r"(size),
        "r"(__as_ptr_smem(smem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// cp.async.bulk.dst.src.mbarrier::complete_tx::bytes [dstMem], [srcMem], size, [rdsmem_bar]; // PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .shared::cta }
template <typename = void>
__device__ static inline void cp_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  void* dstMem,
  const void* srcMem,
  const std::uint32_t& size,
  std::uint64_t* rdsmem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  void* dstMem,
  const void* srcMem,
  const std::uint32_t& size,
  std::uint64_t* rdsmem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];"
      :
      : "r"(__as_ptr_remote_dsmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size),
        "r"(__as_ptr_remote_dsmem(rdsmem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.dst.src.mbarrier::complete_tx::bytes.ignore_oob [dstMem], [srcMem], size, ignoreBytesLeft, ignoreBytesRight, [smem_bar]; // PTX ISA 92, SM_90
// .dst       = { .shared::cta }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk_ignore_oob(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* srcMem,
  const std::uint32_t& size,
  const std::uint32_t& ignoreBytesLeft,
  const std::uint32_t& ignoreBytesRight,
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 920
extern "C" __device__ void __cuda_ptx_cp_async_bulk_ignore_oob_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk_ignore_oob(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* srcMem,
  const std::uint32_t& size,
  const std::uint32_t& ignoreBytesLeft,
  const std::uint32_t& ignoreBytesRight,
  std::uint64_t* smem_bar)
{
  // space == space_shared (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.shared::cta.global.mbarrier::complete_tx::bytes.ignore_oob [%0], [%1], %2, %3, %4, [%5];"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(__as_ptr_gmem(srcMem)),
        "r"(size),
        "r"(ignoreBytesLeft),
        "r"(ignoreBytesRight),
        "r"(__as_ptr_smem(smem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_ignore_oob_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 920

/*
// cp.async.bulk.dst.src.bulk_group [dstMem], [srcMem], size; // PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
template <typename = void>
__device__ static inline void cp_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  void* dstMem,
  const void* srcMem,
  const std::uint32_t& size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  void* dstMem,
  const void* srcMem,
  const std::uint32_t& size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.global.shared::cta.bulk_group [%0], [%1], %2;"
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.dst.src.bulk_group.cp_mask [dstMem], [srcMem], size, byteMask; // PTX ISA 86, SM_100
// .dst       = { .global }
// .src       = { .shared::cta }
template <typename = void>
__device__ static inline void cp_async_bulk_cp_mask(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  void* dstMem,
  const void* srcMem,
  const std::uint32_t& size,
  const std::uint16_t& byteMask);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_cp_async_bulk_cp_mask_is_not_supported_before_SM_100__();
template <typename = void>
__device__ static inline void cp_async_bulk_cp_mask(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  void* dstMem,
  const void* srcMem,
  const std::uint32_t& size,
  const std::uint16_t& byteMask)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    asm (
      "cp.async.bulk.global.shared::cta.bulk_group.cp_mask [%0], [%1], %2, %3;"
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size),
        "h"(byteMask)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_cp_mask_is_not_supported_before_SM_100__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860
/*
// cp.async.bulk.dst.src.mbarrier::complete_tx::bytes.multicast::cluster [dstMem], [srcMem], size, [smem_bar], ctaMask; // PTX ISA 80, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .shared::cluster }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* srcMem,
  const std::uint32_t& size,
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
template <typename = void>
__device__ static inline void cp_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* srcMem,
  const std::uint32_t& size,
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.multicast::cluster [%0], [%1], %2, [%3], %4;"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(__as_ptr_gmem(srcMem)),
        "r"(size),
        "r"(__as_ptr_smem(smem_bar)),
        "h"(ctaMask)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

} // namespace cuda_ptx

#endif // INSTRUCTIONS_CP_ASYNC_BULK_H

// End of ./instructions/cp_async_bulk.h
// Start of ./instructions/cp_async_bulk_commit_group.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_CP_ASYNC_BULK_COMMIT_GROUP_H
#define INSTRUCTIONS_CP_ASYNC_BULK_COMMIT_GROUP_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/cp_async_bulk_commit_group.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// cp.async.bulk.commit_group; // PTX ISA 80, SM_90
template <typename = void>
__device__ static inline void cp_async_bulk_commit_group();
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_commit_group_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk_commit_group()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "cp.async.bulk.commit_group;"
      :
      :
      :
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_commit_group_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

} // namespace cuda_ptx

#endif // INSTRUCTIONS_CP_ASYNC_BULK_COMMIT_GROUP_H

// End of ./instructions/cp_async_bulk_commit_group.h
// Start of ./instructions/cp_async_bulk_prefetch.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_CP_ASYNC_BULK_PREFETCH_H
#define INSTRUCTIONS_CP_ASYNC_BULK_PREFETCH_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/cp_async_bulk_prefetch.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// cp.async.bulk.prefetch.L2.global.L2::cache_hint [srcMem], size, cache_policy; // PTX ISA 80, SM_90
template <typename = void>
__device__ static inline void cp_async_bulk_prefetch(
  const void* srcMem,
  std::uint32_t size,
  std::uint64_t cache_policy = 0x10F0000000000000);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_prefetch_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk_prefetch(
  const void* srcMem,
  std::uint32_t size,
  std::uint64_t cache_policy = 0x10F0000000000000)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "cp.async.bulk.prefetch.L2.global.L2::cache_hint [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(srcMem)),
        "r"(size),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_prefetch_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

} // namespace cuda_ptx

#endif // INSTRUCTIONS_CP_ASYNC_BULK_PREFETCH_H

// End of ./instructions/cp_async_bulk_prefetch.h
// Start of ./instructions/cp_async_bulk_tensor.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_CP_ASYNC_BULK_TENSOR_H
#define INSTRUCTIONS_CP_ASYNC_BULK_TENSOR_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/cp_async_bulk_tensor.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// cp.async.bulk.tensor.1d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; // PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[1],
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[1],
  std::uint64_t* smem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2}], [%3];"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(__as_ptr_smem(smem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.tensor.1d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; // PTX ISA 86, SM_90
// .dst       = { .shared::cta }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[1],
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[1],
  std::uint64_t* smem_bar)
{
  // space == space_shared (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.tensor.1d.shared::cta.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2}], [%3];"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(__as_ptr_smem(smem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// cp.async.bulk.tensor.1d.dst.src.tile.mbarrier::complete_tx::bytes.cta_group [dstMem], [tensorMap, tensorCoords], [smem_bar]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .shared::cta }
// .src       = { .global }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[1],
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[1],
  std::uint64_t* smem_bar)
{
  // space == space_shared (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "cp.async.bulk.tensor.1d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%0], [%1, {%2}], [%3];"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(__as_ptr_smem(smem_bar))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "cp.async.bulk.tensor.1d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%0], [%1, {%2}], [%3];"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(__as_ptr_smem(smem_bar))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// cp.async.bulk.tensor.1d.dst.src.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[1],
  const void* srcMem);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[1],
  const void* srcMem)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.tensor.1d.global.shared::cta.tile.bulk_group [%0, {%1}], [%2];"
      :
      : "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(__as_ptr_smem(srcMem))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.tensor.2d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; // PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[2],
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[2],
  std::uint64_t* smem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3}], [%4];"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(tensorCoords[1]),
        "r"(__as_ptr_smem(smem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.tensor.2d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; // PTX ISA 86, SM_90
// .dst       = { .shared::cta }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[2],
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[2],
  std::uint64_t* smem_bar)
{
  // space == space_shared (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.tensor.2d.shared::cta.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3}], [%4];"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(tensorCoords[1]),
        "r"(__as_ptr_smem(smem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// cp.async.bulk.tensor.2d.dst.src.tile.mbarrier::complete_tx::bytes.cta_group [dstMem], [tensorMap, tensorCoords], [smem_bar]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .shared::cta }
// .src       = { .global }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[2],
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[2],
  std::uint64_t* smem_bar)
{
  // space == space_shared (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "cp.async.bulk.tensor.2d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%0], [%1, {%2, %3}], [%4];"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(__as_ptr_smem(smem_bar))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "cp.async.bulk.tensor.2d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%0], [%1, {%2, %3}], [%4];"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(__as_ptr_smem(smem_bar))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// cp.async.bulk.tensor.2d.dst.src.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[2],
  const void* srcMem);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[2],
  const void* srcMem)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.tensor.2d.global.shared::cta.tile.bulk_group [%0, {%1, %2}], [%3];"
      :
      : "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(tensorCoords[1]),
        "r"(__as_ptr_smem(srcMem))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.tensor.3d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; // PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[3],
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[3],
  std::uint64_t* smem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3, %4}], [%5];"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(tensorCoords[1]),
        "r"(tensorCoords[2]),
        "r"(__as_ptr_smem(smem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.tensor.3d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; // PTX ISA 86, SM_90
// .dst       = { .shared::cta }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[3],
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[3],
  std::uint64_t* smem_bar)
{
  // space == space_shared (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.tensor.3d.shared::cta.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3, %4}], [%5];"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(tensorCoords[1]),
        "r"(tensorCoords[2]),
        "r"(__as_ptr_smem(smem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// cp.async.bulk.tensor.3d.dst.src.tile.mbarrier::complete_tx::bytes.cta_group [dstMem], [tensorMap, tensorCoords], [smem_bar]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .shared::cta }
// .src       = { .global }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[3],
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[3],
  std::uint64_t* smem_bar)
{
  // space == space_shared (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "cp.async.bulk.tensor.3d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%0], [%1, {%2, %3, %4}], [%5];"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(__as_ptr_smem(smem_bar))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "cp.async.bulk.tensor.3d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%0], [%1, {%2, %3, %4}], [%5];"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(__as_ptr_smem(smem_bar))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// cp.async.bulk.tensor.3d.dst.src.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[3],
  const void* srcMem);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[3],
  const void* srcMem)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.tensor.3d.global.shared::cta.tile.bulk_group [%0, {%1, %2, %3}], [%4];"
      :
      : "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(tensorCoords[1]),
        "r"(tensorCoords[2]),
        "r"(__as_ptr_smem(srcMem))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.tensor.4d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; // PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[4],
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[4],
  std::uint64_t* smem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3, %4, %5}], [%6];"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(tensorCoords[1]),
        "r"(tensorCoords[2]),
        "r"(tensorCoords[3]),
        "r"(__as_ptr_smem(smem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.tensor.4d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; // PTX ISA 86, SM_90
// .dst       = { .shared::cta }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[4],
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[4],
  std::uint64_t* smem_bar)
{
  // space == space_shared (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.tensor.4d.shared::cta.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3, %4, %5}], [%6];"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(tensorCoords[1]),
        "r"(tensorCoords[2]),
        "r"(tensorCoords[3]),
        "r"(__as_ptr_smem(smem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// cp.async.bulk.tensor.4d.dst.src.tile.mbarrier::complete_tx::bytes.cta_group [dstMem], [tensorMap, tensorCoords], [smem_bar]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .shared::cta }
// .src       = { .global }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[4],
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[4],
  std::uint64_t* smem_bar)
{
  // space == space_shared (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "cp.async.bulk.tensor.4d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%0], [%1, {%2, %3, %4, %5}], [%6];"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(__as_ptr_smem(smem_bar))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "cp.async.bulk.tensor.4d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%0], [%1, {%2, %3, %4, %5}], [%6];"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(__as_ptr_smem(smem_bar))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// cp.async.bulk.tensor.4d.dst.src.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[4],
  const void* srcMem);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[4],
  const void* srcMem)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.tensor.4d.global.shared::cta.tile.bulk_group [%0, {%1, %2, %3, %4}], [%5];"
      :
      : "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(tensorCoords[1]),
        "r"(tensorCoords[2]),
        "r"(tensorCoords[3]),
        "r"(__as_ptr_smem(srcMem))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.tensor.5d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; // PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  std::uint64_t* smem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3, %4, %5, %6}], [%7];"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(tensorCoords[1]),
        "r"(tensorCoords[2]),
        "r"(tensorCoords[3]),
        "r"(tensorCoords[4]),
        "r"(__as_ptr_smem(smem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.tensor.5d.dst.src.tile.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; // PTX ISA 86, SM_90
// .dst       = { .shared::cta }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  std::uint64_t* smem_bar)
{
  // space == space_shared (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.tensor.5d.shared::cta.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3, %4, %5, %6}], [%7];"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(tensorCoords[1]),
        "r"(tensorCoords[2]),
        "r"(tensorCoords[3]),
        "r"(tensorCoords[4]),
        "r"(__as_ptr_smem(smem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// cp.async.bulk.tensor.5d.dst.src.tile.mbarrier::complete_tx::bytes.cta_group [dstMem], [tensorMap, tensorCoords], [smem_bar]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .shared::cta }
// .src       = { .global }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  std::uint64_t* smem_bar)
{
  // space == space_shared (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "cp.async.bulk.tensor.5d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::1 [%0], [%1, {%2, %3, %4, %5, %6}], [%7];"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(tensorCoords[4]),
          "r"(__as_ptr_smem(smem_bar))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "cp.async.bulk.tensor.5d.shared::cta.global.tile.mbarrier::complete_tx::bytes.cta_group::2 [%0], [%1, {%2, %3, %4, %5, %6}], [%7];"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(tensorCoords[4]),
          "r"(__as_ptr_smem(smem_bar))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// cp.async.bulk.tensor.5d.dst.src.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  const void* srcMem);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  const void* srcMem)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.async.bulk.tensor.5d.global.shared::cta.tile.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6];"
      :
      : "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(tensorCoords[1]),
        "r"(tensorCoords[2]),
        "r"(tensorCoords[3]),
        "r"(tensorCoords[4]),
        "r"(__as_ptr_smem(srcMem))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800
/*
// cp.async.bulk.tensor.1d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster [dstMem], [tensorMap, tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .shared::cluster }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[1],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[1],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster [%0], [%1, {%2}], [%3], %4;"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(__as_ptr_smem(smem_bar)),
        "h"(ctaMask)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.tensor.1d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group [dstMem], [tensorMap, tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .shared::cluster }
// .src       = { .global }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[1],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[1],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%0], [%1, {%2}], [%3], %4;"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(__as_ptr_smem(smem_bar)),
          "h"(ctaMask)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "cp.async.bulk.tensor.1d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%0], [%1, {%2}], [%3], %4;"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(__as_ptr_smem(smem_bar)),
          "h"(ctaMask)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.tensor.2d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster [dstMem], [tensorMap, tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .shared::cluster }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[2],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[2],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster [%0], [%1, {%2, %3}], [%4], %5;"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(tensorCoords[1]),
        "r"(__as_ptr_smem(smem_bar)),
        "h"(ctaMask)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.tensor.2d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group [dstMem], [tensorMap, tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .shared::cluster }
// .src       = { .global }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[2],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[2],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%0], [%1, {%2, %3}], [%4], %5;"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(__as_ptr_smem(smem_bar)),
          "h"(ctaMask)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "cp.async.bulk.tensor.2d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%0], [%1, {%2, %3}], [%4], %5;"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(__as_ptr_smem(smem_bar)),
          "h"(ctaMask)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.tensor.3d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster [dstMem], [tensorMap, tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .shared::cluster }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[3],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[3],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster [%0], [%1, {%2, %3, %4}], [%5], %6;"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(tensorCoords[1]),
        "r"(tensorCoords[2]),
        "r"(__as_ptr_smem(smem_bar)),
        "h"(ctaMask)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.tensor.3d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group [dstMem], [tensorMap, tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .shared::cluster }
// .src       = { .global }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[3],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[3],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%0], [%1, {%2, %3, %4}], [%5], %6;"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(__as_ptr_smem(smem_bar)),
          "h"(ctaMask)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "cp.async.bulk.tensor.3d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%0], [%1, {%2, %3, %4}], [%5], %6;"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(__as_ptr_smem(smem_bar)),
          "h"(ctaMask)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.tensor.4d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster [dstMem], [tensorMap, tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .shared::cluster }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[4],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[4],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster [%0], [%1, {%2, %3, %4, %5}], [%6], %7;"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(tensorCoords[1]),
        "r"(tensorCoords[2]),
        "r"(tensorCoords[3]),
        "r"(__as_ptr_smem(smem_bar)),
        "h"(ctaMask)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.tensor.4d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group [dstMem], [tensorMap, tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .shared::cluster }
// .src       = { .global }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[4],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[4],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%0], [%1, {%2, %3, %4, %5}], [%6], %7;"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(__as_ptr_smem(smem_bar)),
          "h"(ctaMask)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "cp.async.bulk.tensor.4d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%0], [%1, {%2, %3, %4, %5}], [%6], %7;"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(__as_ptr_smem(smem_bar)),
          "h"(ctaMask)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.tensor.5d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster [dstMem], [tensorMap, tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .shared::cluster }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster [%0], [%1, {%2, %3, %4, %5, %6}], [%7], %8;"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(tensorCoords[1]),
        "r"(tensorCoords[2]),
        "r"(tensorCoords[3]),
        "r"(tensorCoords[4]),
        "r"(__as_ptr_smem(smem_bar)),
        "h"(ctaMask)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.tensor.5d.dst.src.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group [dstMem], [tensorMap, tensorCoords], [smem_bar], ctaMask; // PTX ISA 80, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .shared::cluster }
// .src       = { .global }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%0], [%1, {%2, %3, %4, %5, %6}], [%7], %8;"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(tensorCoords[4]),
          "r"(__as_ptr_smem(smem_bar)),
          "h"(ctaMask)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "cp.async.bulk.tensor.5d.shared::cluster.global.tile.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%0], [%1, {%2, %3, %4, %5, %6}], [%7], %8;"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(tensorCoords[4]),
          "r"(__as_ptr_smem(smem_bar)),
          "h"(ctaMask)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800
/*
// cp.async.bulk.tensor.2d.dst.src.tile::gather4.mbarrier::complete_tx::bytes [dstMem], [tensorMap, tensorCoords], [smem_bar]; // PTX ISA 86, SM_100
// .dst       = { .shared::cta }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor_tile_gather4(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_tile_gather4_is_not_supported_before_SM_100__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor_tile_gather4(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  std::uint64_t* smem_bar)
{
  // space == space_shared (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    asm (
      "cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3, %4, %5, %6}], [%7];"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(tensorCoords[1]),
        "r"(tensorCoords[2]),
        "r"(tensorCoords[3]),
        "r"(tensorCoords[4]),
        "r"(__as_ptr_smem(smem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_tile_gather4_is_not_supported_before_SM_100__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// cp.async.bulk.tensor.2d.dst.src.tile::gather4.mbarrier::complete_tx::bytes.cta_group [dstMem], [tensorMap, tensorCoords], [smem_bar]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .shared::cta }
// .src       = { .global }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor_tile_gather4(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_tile_gather4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor_tile_gather4(
  cuda_ptx::space_shared_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  std::uint64_t* smem_bar)
{
  // space == space_shared (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes.cta_group::1 [%0], [%1, {%2, %3, %4, %5, %6}], [%7];"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(tensorCoords[4]),
          "r"(__as_ptr_smem(smem_bar))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes.cta_group::2 [%0], [%1, {%2, %3, %4, %5, %6}], [%7];"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(tensorCoords[4]),
          "r"(__as_ptr_smem(smem_bar))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_tile_gather4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// cp.async.bulk.tensor.2d.dst.src.tile::gather4.mbarrier::complete_tx::bytes.multicast::cluster [dstMem], [tensorMap, tensorCoords], [smem_bar], ctaMask; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .shared::cluster }
// .src       = { .global }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor_tile_gather4(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_tile_gather4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor_tile_gather4(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "cp.async.bulk.tensor.2d.shared::cluster.global.tile::gather4.mbarrier::complete_tx::bytes.multicast::cluster [%0], [%1, {%2, %3, %4, %5, %6}], [%7], %8;"
      :
      : "r"(__as_ptr_smem(dstMem)),
        "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(tensorCoords[1]),
        "r"(tensorCoords[2]),
        "r"(tensorCoords[3]),
        "r"(tensorCoords[4]),
        "r"(__as_ptr_smem(smem_bar)),
        "h"(ctaMask)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_tile_gather4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// cp.async.bulk.tensor.2d.dst.src.tile::gather4.mbarrier::complete_tx::bytes.multicast::cluster.cta_group [dstMem], [tensorMap, tensorCoords], [smem_bar], ctaMask; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .shared::cluster }
// .src       = { .global }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor_tile_gather4(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_tile_gather4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void cp_async_bulk_tensor_tile_gather4(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_global_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  void* dstMem,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  std::uint64_t* smem_bar,
  const std::uint16_t& ctaMask)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_global (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "cp.async.bulk.tensor.2d.shared::cluster.global.tile::gather4.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::1 [%0], [%1, {%2, %3, %4, %5, %6}], [%7], %8;"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(tensorCoords[4]),
          "r"(__as_ptr_smem(smem_bar)),
          "h"(ctaMask)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "cp.async.bulk.tensor.2d.shared::cluster.global.tile::gather4.mbarrier::complete_tx::bytes.multicast::cluster.cta_group::2 [%0], [%1, {%2, %3, %4, %5, %6}], [%7], %8;"
        :
        : "r"(__as_ptr_smem(dstMem)),
          "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(tensorCoords[4]),
          "r"(__as_ptr_smem(smem_bar)),
          "h"(ctaMask)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_tile_gather4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// cp.async.bulk.tensor.2d.dst.src.tile::scatter4.bulk_group [tensorMap, tensorCoords], [srcMem]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .dst       = { .global }
// .src       = { .shared::cta }
template <typename = void>
__device__ static inline void cp_async_bulk_tensor_tile_scatter4(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  const void* srcMem);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_cp_async_bulk_tensor_tile_scatter4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename = void>
__device__ static inline void cp_async_bulk_tensor_tile_scatter4(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  const void* srcMem)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "cp.async.bulk.tensor.2d.global.shared::cta.tile::scatter4.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6];"
      :
      : "l"(tensorMap),
        "r"(tensorCoords[0]),
        "r"(tensorCoords[1]),
        "r"(tensorCoords[2]),
        "r"(tensorCoords[3]),
        "r"(tensorCoords[4]),
        "r"(__as_ptr_smem(srcMem))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_tensor_tile_scatter4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

} // namespace cuda_ptx

#endif // INSTRUCTIONS_CP_ASYNC_BULK_TENSOR_H

// End of ./instructions/cp_async_bulk_tensor.h
// Start of ./instructions/cp_async_bulk_wait_group.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_CP_ASYNC_BULK_WAIT_GROUP_H
#define INSTRUCTIONS_CP_ASYNC_BULK_WAIT_GROUP_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/cp_async_bulk_wait_group.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// cp.async.bulk.wait_group N; // PTX ISA 80, SM_90
template <int N32>
__device__ static inline void cp_async_bulk_wait_group(
  cuda_ptx::n32_t<N32> N);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_wait_group_is_not_supported_before_SM_90__();
template <int N32>
__device__ static inline void cp_async_bulk_wait_group(
  cuda_ptx::n32_t<N32> N)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "cp.async.bulk.wait_group %0;"
      :
      : "n"(N.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_wait_group_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.async.bulk.wait_group.read N; // PTX ISA 80, SM_90
template <int N32>
__device__ static inline void cp_async_bulk_wait_group_read(
  cuda_ptx::n32_t<N32> N);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_async_bulk_wait_group_read_is_not_supported_before_SM_90__();
template <int N32>
__device__ static inline void cp_async_bulk_wait_group_read(
  cuda_ptx::n32_t<N32> N)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "cp.async.bulk.wait_group.read %0;"
      :
      : "n"(N.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_bulk_wait_group_read_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

} // namespace cuda_ptx

#endif // INSTRUCTIONS_CP_ASYNC_BULK_WAIT_GROUP_H

// End of ./instructions/cp_async_bulk_wait_group.h
// Start of ./instructions/cp_async_mbarrier_arrive.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_CP_ASYNC_MBARRIER_ARRIVE_H
#define INSTRUCTIONS_CP_ASYNC_MBARRIER_ARRIVE_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/cp_async_mbarrier_arrive.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// cp.async.mbarrier.arrive.b64 [addr]; // PTX ISA 70, SM_80
template <typename = void>
__device__ static inline void cp_async_mbarrier_arrive(
  std::uint64_t* addr);
*/
#if __libcuda_ptx_isa >= 700
extern "C" __device__ void __cuda_ptx_cp_async_mbarrier_arrive_is_not_supported_before_SM_80__();
template <typename = void>
__device__ static inline void cp_async_mbarrier_arrive(
  std::uint64_t* addr)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "cp.async.mbarrier.arrive.b64 [%0];"
      :
      : "r"(__as_ptr_smem(addr))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_async_mbarrier_arrive_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 700

} // namespace cuda_ptx

#endif // INSTRUCTIONS_CP_ASYNC_MBARRIER_ARRIVE_H

// End of ./instructions/cp_async_mbarrier_arrive.h
// Start of ./instructions/cp_reduce_async_bulk.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_CP_REDUCE_ASYNC_BULK_H
#define INSTRUCTIONS_CP_REDUCE_ASYNC_BULK_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/cp_reduce_async_bulk.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// cp.reduce.async.bulk.dst.src.mbarrier::complete_tx::bytes.op.type [dstMem], [srcMem], size, [rdsmem_bar]; // 1. PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .shared::cta }
// .type      = { .b32 }
// .op        = { .and }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_and_op_t,
  B32* dstMem,
  const B32* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_and_op_t,
  B32* dstMem,
  const B32* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_b32 (due to parameter type constraint)
  // op == op_and_op (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes.and.b32 [%0], [%1], %2, [%3]; // 1."
      :
      : "r"(__as_ptr_remote_dsmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size),
        "r"(__as_ptr_remote_dsmem(rdsmem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.mbarrier::complete_tx::bytes.op.type [dstMem], [srcMem], size, [rdsmem_bar]; // 1. PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .shared::cta }
// .type      = { .b32 }
// .op        = { .or }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_or_op_t,
  B32* dstMem,
  const B32* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_or_op_t,
  B32* dstMem,
  const B32* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_b32 (due to parameter type constraint)
  // op == op_or_op (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes.or.b32 [%0], [%1], %2, [%3]; // 1."
      :
      : "r"(__as_ptr_remote_dsmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size),
        "r"(__as_ptr_remote_dsmem(rdsmem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.mbarrier::complete_tx::bytes.op.type [dstMem], [srcMem], size, [rdsmem_bar]; // 1. PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .shared::cta }
// .type      = { .b32 }
// .op        = { .xor }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_xor_op_t,
  B32* dstMem,
  const B32* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_xor_op_t,
  B32* dstMem,
  const B32* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_b32 (due to parameter type constraint)
  // op == op_xor_op (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes.xor.b32 [%0], [%1], %2, [%3]; // 1."
      :
      : "r"(__as_ptr_remote_dsmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size),
        "r"(__as_ptr_remote_dsmem(rdsmem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.mbarrier::complete_tx::bytes.op.type [dstMem], [srcMem], size, [rdsmem_bar]; // 1. PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .shared::cta }
// .type      = { .u32 }
// .op        = { .min }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_min_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_min_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_u32 (due to parameter type constraint)
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes.min.u32 [%0], [%1], %2, [%3]; // 1."
      :
      : "r"(__as_ptr_remote_dsmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size),
        "r"(__as_ptr_remote_dsmem(rdsmem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.mbarrier::complete_tx::bytes.op.type [dstMem], [srcMem], size, [rdsmem_bar]; // 1. PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .shared::cta }
// .type      = { .u32 }
// .op        = { .max }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_max_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_max_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_u32 (due to parameter type constraint)
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes.max.u32 [%0], [%1], %2, [%3]; // 1."
      :
      : "r"(__as_ptr_remote_dsmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size),
        "r"(__as_ptr_remote_dsmem(rdsmem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.mbarrier::complete_tx::bytes.op.type [dstMem], [srcMem], size, [rdsmem_bar]; // 1. PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .shared::cta }
// .type      = { .u32 }
// .op        = { .add }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_u32 (due to parameter type constraint)
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes.add.u32 [%0], [%1], %2, [%3]; // 1."
      :
      : "r"(__as_ptr_remote_dsmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size),
        "r"(__as_ptr_remote_dsmem(rdsmem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.mbarrier::complete_tx::bytes.op.type [dstMem], [srcMem], size, [rdsmem_bar]; // 1. PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .shared::cta }
// .type      = { .u32 }
// .op        = { .inc }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_inc_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_inc_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_u32 (due to parameter type constraint)
  // op == op_inc (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes.inc.u32 [%0], [%1], %2, [%3]; // 1."
      :
      : "r"(__as_ptr_remote_dsmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size),
        "r"(__as_ptr_remote_dsmem(rdsmem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.mbarrier::complete_tx::bytes.op.type [dstMem], [srcMem], size, [rdsmem_bar]; // 1. PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .shared::cta }
// .type      = { .u32 }
// .op        = { .dec }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_dec_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_dec_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_u32 (due to parameter type constraint)
  // op == op_dec (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes.dec.u32 [%0], [%1], %2, [%3]; // 1."
      :
      : "r"(__as_ptr_remote_dsmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size),
        "r"(__as_ptr_remote_dsmem(rdsmem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.mbarrier::complete_tx::bytes.op.type [dstMem], [srcMem], size, [rdsmem_bar]; // 1. PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .shared::cta }
// .type      = { .s32 }
// .op        = { .min }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_min_t,
  std::int32_t* dstMem,
  const std::int32_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_min_t,
  std::int32_t* dstMem,
  const std::int32_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_s32 (due to parameter type constraint)
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes.min.s32 [%0], [%1], %2, [%3]; // 1."
      :
      : "r"(__as_ptr_remote_dsmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size),
        "r"(__as_ptr_remote_dsmem(rdsmem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.mbarrier::complete_tx::bytes.op.type [dstMem], [srcMem], size, [rdsmem_bar]; // 1. PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .shared::cta }
// .type      = { .s32 }
// .op        = { .max }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_max_t,
  std::int32_t* dstMem,
  const std::int32_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_max_t,
  std::int32_t* dstMem,
  const std::int32_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_s32 (due to parameter type constraint)
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes.max.s32 [%0], [%1], %2, [%3]; // 1."
      :
      : "r"(__as_ptr_remote_dsmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size),
        "r"(__as_ptr_remote_dsmem(rdsmem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.mbarrier::complete_tx::bytes.op.type [dstMem], [srcMem], size, [rdsmem_bar]; // 1. PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .shared::cta }
// .type      = { .s32 }
// .op        = { .add }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  std::int32_t* dstMem,
  const std::int32_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  std::int32_t* dstMem,
  const std::int32_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_s32 (due to parameter type constraint)
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes.add.s32 [%0], [%1], %2, [%3]; // 1."
      :
      : "r"(__as_ptr_remote_dsmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size),
        "r"(__as_ptr_remote_dsmem(rdsmem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.mbarrier::complete_tx::bytes.op.type [dstMem], [srcMem], size, [rdsmem_bar]; // 1. PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .shared::cta }
// .type      = { .u64 }
// .op        = { .add }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  std::uint64_t* dstMem,
  const std::uint64_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  std::uint64_t* dstMem,
  const std::uint64_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_u64 (due to parameter type constraint)
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes.add.u64 [%0], [%1], %2, [%3]; // 1."
      :
      : "r"(__as_ptr_remote_dsmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size),
        "r"(__as_ptr_remote_dsmem(rdsmem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.mbarrier::complete_tx::bytes.op.u64 [dstMem], [srcMem], size, [rdsmem_bar]; // 2. PTX ISA 80, SM_90
// .dst       = { .shared::cluster }
// .src       = { .shared::cta }
// .type      = { .s64 }
// .op        = { .add }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  std::int64_t* dstMem,
  const std::int64_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_cluster_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  std::int64_t* dstMem,
  const std::int64_t* srcMem,
  std::uint32_t size,
  std::uint64_t* rdsmem_bar)
{
  // space == space_cluster (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_s64 (due to parameter type constraint)
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.shared::cluster.shared::cta.mbarrier::complete_tx::bytes.add.u64 [%0], [%1], %2, [%3]; // 2."
      :
      : "r"(__as_ptr_remote_dsmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size),
        "r"(__as_ptr_remote_dsmem(rdsmem_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 3. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .b32, .b64 }
// .op        = { .and }
template <typename Type>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_and_op_t,
  Type* dstMem,
  const Type* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename Type>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_and_op_t,
  Type* dstMem,
  const Type* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  static_assert(sizeof(Type) == 4 || sizeof(Type) == 8, "");
  // op == op_and_op (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sizeof(Type) == 4) {
      asm (
        "cp.reduce.async.bulk.global.shared::cta.bulk_group.and.b32  [%0], [%1], %2; // 3."
        :
        : "l"(__as_ptr_gmem(dstMem)),
          "r"(__as_ptr_smem(srcMem)),
          "r"(size)
        : "memory"
      );
    } else if constexpr (sizeof(Type) == 8) {
      asm (
        "cp.reduce.async.bulk.global.shared::cta.bulk_group.and.b64  [%0], [%1], %2; // 3."
        :
        : "l"(__as_ptr_gmem(dstMem)),
          "r"(__as_ptr_smem(srcMem)),
          "r"(size)
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 3. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .b32, .b64 }
// .op        = { .or }
template <typename Type>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_or_op_t,
  Type* dstMem,
  const Type* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename Type>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_or_op_t,
  Type* dstMem,
  const Type* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  static_assert(sizeof(Type) == 4 || sizeof(Type) == 8, "");
  // op == op_or_op (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sizeof(Type) == 4) {
      asm (
        "cp.reduce.async.bulk.global.shared::cta.bulk_group.or.b32  [%0], [%1], %2; // 3."
        :
        : "l"(__as_ptr_gmem(dstMem)),
          "r"(__as_ptr_smem(srcMem)),
          "r"(size)
        : "memory"
      );
    } else if constexpr (sizeof(Type) == 8) {
      asm (
        "cp.reduce.async.bulk.global.shared::cta.bulk_group.or.b64  [%0], [%1], %2; // 3."
        :
        : "l"(__as_ptr_gmem(dstMem)),
          "r"(__as_ptr_smem(srcMem)),
          "r"(size)
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 3. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .b32, .b64 }
// .op        = { .xor }
template <typename Type>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_xor_op_t,
  Type* dstMem,
  const Type* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename Type>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_xor_op_t,
  Type* dstMem,
  const Type* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  static_assert(sizeof(Type) == 4 || sizeof(Type) == 8, "");
  // op == op_xor_op (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sizeof(Type) == 4) {
      asm (
        "cp.reduce.async.bulk.global.shared::cta.bulk_group.xor.b32  [%0], [%1], %2; // 3."
        :
        : "l"(__as_ptr_gmem(dstMem)),
          "r"(__as_ptr_smem(srcMem)),
          "r"(size)
        : "memory"
      );
    } else if constexpr (sizeof(Type) == 8) {
      asm (
        "cp.reduce.async.bulk.global.shared::cta.bulk_group.xor.b64  [%0], [%1], %2; // 3."
        :
        : "l"(__as_ptr_gmem(dstMem)),
          "r"(__as_ptr_smem(srcMem)),
          "r"(size)
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .u32 }
// .op        = { .min }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_min_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_min_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_u32 (due to parameter type constraint)
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.min.u32  [%0], [%1], %2; // 4."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .u32 }
// .op        = { .max }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_max_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_max_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_u32 (due to parameter type constraint)
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.max.u32  [%0], [%1], %2; // 4."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .u32 }
// .op        = { .add }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_u32 (due to parameter type constraint)
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.add.u32  [%0], [%1], %2; // 4."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .u32 }
// .op        = { .inc }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_inc_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_inc_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_u32 (due to parameter type constraint)
  // op == op_inc (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.inc.u32  [%0], [%1], %2; // 4."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .u32 }
// .op        = { .dec }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_dec_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_dec_t,
  std::uint32_t* dstMem,
  const std::uint32_t* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_u32 (due to parameter type constraint)
  // op == op_dec (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.dec.u32  [%0], [%1], %2; // 4."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .s32 }
// .op        = { .min }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_min_t,
  std::int32_t* dstMem,
  const std::int32_t* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_min_t,
  std::int32_t* dstMem,
  const std::int32_t* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_s32 (due to parameter type constraint)
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.min.s32  [%0], [%1], %2; // 4."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .s32 }
// .op        = { .max }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_max_t,
  std::int32_t* dstMem,
  const std::int32_t* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_max_t,
  std::int32_t* dstMem,
  const std::int32_t* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_s32 (due to parameter type constraint)
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.max.s32  [%0], [%1], %2; // 4."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .s32 }
// .op        = { .add }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  std::int32_t* dstMem,
  const std::int32_t* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  std::int32_t* dstMem,
  const std::int32_t* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_s32 (due to parameter type constraint)
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.add.s32  [%0], [%1], %2; // 4."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .u64 }
// .op        = { .min }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_min_t,
  std::uint64_t* dstMem,
  const std::uint64_t* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_min_t,
  std::uint64_t* dstMem,
  const std::uint64_t* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_u64 (due to parameter type constraint)
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.min.u64  [%0], [%1], %2; // 4."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .u64 }
// .op        = { .max }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_max_t,
  std::uint64_t* dstMem,
  const std::uint64_t* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_max_t,
  std::uint64_t* dstMem,
  const std::uint64_t* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_u64 (due to parameter type constraint)
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.max.u64  [%0], [%1], %2; // 4."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .u64 }
// .op        = { .add }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  std::uint64_t* dstMem,
  const std::uint64_t* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  std::uint64_t* dstMem,
  const std::uint64_t* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_u64 (due to parameter type constraint)
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.add.u64  [%0], [%1], %2; // 4."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .s64 }
// .op        = { .min }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_min_t,
  std::int64_t* dstMem,
  const std::int64_t* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_min_t,
  std::int64_t* dstMem,
  const std::int64_t* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_s64 (due to parameter type constraint)
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.min.s64  [%0], [%1], %2; // 4."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .s64 }
// .op        = { .max }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_max_t,
  std::int64_t* dstMem,
  const std::int64_t* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_max_t,
  std::int64_t* dstMem,
  const std::int64_t* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_s64 (due to parameter type constraint)
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.max.s64  [%0], [%1], %2; // 4."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .f32 }
// .op        = { .add }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  float* dstMem,
  const float* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  float* dstMem,
  const float* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_f32 (due to parameter type constraint)
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.add.f32  [%0], [%1], %2; // 4."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .f64 }
// .op        = { .add }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  double* dstMem,
  const double* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  double* dstMem,
  const double* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_f64 (due to parameter type constraint)
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.add.f64  [%0], [%1], %2; // 4."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.u64  [dstMem], [srcMem], size; // 6. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .s64 }
// .op        = { .add }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  std::int64_t* dstMem,
  const std::int64_t* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  std::int64_t* dstMem,
  const std::int64_t* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_s64 (due to parameter type constraint)
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.add.u64  [%0], [%1], %2; // 6."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800
/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .f16 }
// .op        = { .min }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_min_t,
  __half* dstMem,
  const __half* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_min_t,
  __half* dstMem,
  const __half* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_f16 (due to parameter type constraint)
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.min.f16  [%0], [%1], %2; // 4."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .f16 }
// .op        = { .max }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_max_t,
  __half* dstMem,
  const __half* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_max_t,
  __half* dstMem,
  const __half* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_f16 (due to parameter type constraint)
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.max.f16  [%0], [%1], %2; // 4."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.noftz.type  [dstMem], [srcMem], size; // 5. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .f16 }
// .op        = { .add }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  __half* dstMem,
  const __half* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  __half* dstMem,
  const __half* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_f16 (due to parameter type constraint)
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.add.noftz.f16  [%0], [%1], %2; // 5."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800
/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .bf16 }
// .op        = { .min }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_min_t,
  __nv_bfloat16* dstMem,
  const __nv_bfloat16* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_min_t,
  __nv_bfloat16* dstMem,
  const __nv_bfloat16* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_bf16 (due to parameter type constraint)
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.min.bf16  [%0], [%1], %2; // 4."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.type  [dstMem], [srcMem], size; // 4. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .bf16 }
// .op        = { .max }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_max_t,
  __nv_bfloat16* dstMem,
  const __nv_bfloat16* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_max_t,
  __nv_bfloat16* dstMem,
  const __nv_bfloat16* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_bf16 (due to parameter type constraint)
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.max.bf16  [%0], [%1], %2; // 4."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.dst.src.bulk_group.op.noftz.type  [dstMem], [srcMem], size; // 5. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .type      = { .bf16 }
// .op        = { .add }
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  __nv_bfloat16* dstMem,
  const __nv_bfloat16* srcMem,
  std::uint32_t size);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void cp_reduce_async_bulk(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_add_t,
  __nv_bfloat16* dstMem,
  const __nv_bfloat16* srcMem,
  std::uint32_t size)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // type == type_bf16 (due to parameter type constraint)
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "cp.reduce.async.bulk.global.shared::cta.bulk_group.add.noftz.bf16  [%0], [%1], %2; // 5."
      :
      : "l"(__as_ptr_gmem(dstMem)),
        "r"(__as_ptr_smem(srcMem)),
        "r"(size)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

} // namespace cuda_ptx

#endif // INSTRUCTIONS_CP_REDUCE_ASYNC_BULK_H

// End of ./instructions/cp_reduce_async_bulk.h
// Start of ./instructions/cp_reduce_async_bulk_tensor.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_CP_REDUCE_ASYNC_BULK_TENSOR_H
#define INSTRUCTIONS_CP_REDUCE_ASYNC_BULK_TENSOR_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/cp_reduce_async_bulk_tensor.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// cp.reduce.async.bulk.tensor.1d.dst.src.op.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // 1a. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .op        = { .add, .min, .max, .inc, .dec, .and, .or, .xor }
template <cuda_ptx::dot_op Op>
__device__ static inline void cp_reduce_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_t<Op> op,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[1],
  const void* srcMem);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_op Op>
__device__ static inline void cp_reduce_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_t<Op> op,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[1],
  const void* srcMem)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  static_assert(op == op_add || op == op_min || op == op_max || op == op_inc || op == op_dec || op == op_and_op || op == op_or_op || op == op_xor_op, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (op == op_add) {
      asm (
        "cp.reduce.async.bulk.tensor.1d.global.shared::cta.add.tile.bulk_group [%0, {%1}], [%2]; // 1a."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_min) {
      asm (
        "cp.reduce.async.bulk.tensor.1d.global.shared::cta.min.tile.bulk_group [%0, {%1}], [%2]; // 1a."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "cp.reduce.async.bulk.tensor.1d.global.shared::cta.max.tile.bulk_group [%0, {%1}], [%2]; // 1a."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_inc) {
      asm (
        "cp.reduce.async.bulk.tensor.1d.global.shared::cta.inc.tile.bulk_group [%0, {%1}], [%2]; // 1a."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_dec) {
      asm (
        "cp.reduce.async.bulk.tensor.1d.global.shared::cta.dec.tile.bulk_group [%0, {%1}], [%2]; // 1a."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_and_op) {
      asm (
        "cp.reduce.async.bulk.tensor.1d.global.shared::cta.and.tile.bulk_group [%0, {%1}], [%2]; // 1a."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_or_op) {
      asm (
        "cp.reduce.async.bulk.tensor.1d.global.shared::cta.or.tile.bulk_group [%0, {%1}], [%2]; // 1a."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_xor_op) {
      asm (
        "cp.reduce.async.bulk.tensor.1d.global.shared::cta.xor.tile.bulk_group [%0, {%1}], [%2]; // 1a."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.tensor.2d.dst.src.op.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // 1b. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .op        = { .add, .min, .max, .inc, .dec, .and, .or, .xor }
template <cuda_ptx::dot_op Op>
__device__ static inline void cp_reduce_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_t<Op> op,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[2],
  const void* srcMem);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_op Op>
__device__ static inline void cp_reduce_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_t<Op> op,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[2],
  const void* srcMem)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  static_assert(op == op_add || op == op_min || op == op_max || op == op_inc || op == op_dec || op == op_and_op || op == op_or_op || op == op_xor_op, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (op == op_add) {
      asm (
        "cp.reduce.async.bulk.tensor.2d.global.shared::cta.add.tile.bulk_group [%0, {%1, %2}], [%3]; // 1b."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_min) {
      asm (
        "cp.reduce.async.bulk.tensor.2d.global.shared::cta.min.tile.bulk_group [%0, {%1, %2}], [%3]; // 1b."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "cp.reduce.async.bulk.tensor.2d.global.shared::cta.max.tile.bulk_group [%0, {%1, %2}], [%3]; // 1b."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_inc) {
      asm (
        "cp.reduce.async.bulk.tensor.2d.global.shared::cta.inc.tile.bulk_group [%0, {%1, %2}], [%3]; // 1b."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_dec) {
      asm (
        "cp.reduce.async.bulk.tensor.2d.global.shared::cta.dec.tile.bulk_group [%0, {%1, %2}], [%3]; // 1b."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_and_op) {
      asm (
        "cp.reduce.async.bulk.tensor.2d.global.shared::cta.and.tile.bulk_group [%0, {%1, %2}], [%3]; // 1b."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_or_op) {
      asm (
        "cp.reduce.async.bulk.tensor.2d.global.shared::cta.or.tile.bulk_group [%0, {%1, %2}], [%3]; // 1b."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_xor_op) {
      asm (
        "cp.reduce.async.bulk.tensor.2d.global.shared::cta.xor.tile.bulk_group [%0, {%1, %2}], [%3]; // 1b."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.tensor.3d.dst.src.op.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // 1c. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .op        = { .add, .min, .max, .inc, .dec, .and, .or, .xor }
template <cuda_ptx::dot_op Op>
__device__ static inline void cp_reduce_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_t<Op> op,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[3],
  const void* srcMem);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_op Op>
__device__ static inline void cp_reduce_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_t<Op> op,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[3],
  const void* srcMem)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  static_assert(op == op_add || op == op_min || op == op_max || op == op_inc || op == op_dec || op == op_and_op || op == op_or_op || op == op_xor_op, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (op == op_add) {
      asm (
        "cp.reduce.async.bulk.tensor.3d.global.shared::cta.add.tile.bulk_group [%0, {%1, %2, %3}], [%4]; // 1c."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_min) {
      asm (
        "cp.reduce.async.bulk.tensor.3d.global.shared::cta.min.tile.bulk_group [%0, {%1, %2, %3}], [%4]; // 1c."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "cp.reduce.async.bulk.tensor.3d.global.shared::cta.max.tile.bulk_group [%0, {%1, %2, %3}], [%4]; // 1c."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_inc) {
      asm (
        "cp.reduce.async.bulk.tensor.3d.global.shared::cta.inc.tile.bulk_group [%0, {%1, %2, %3}], [%4]; // 1c."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_dec) {
      asm (
        "cp.reduce.async.bulk.tensor.3d.global.shared::cta.dec.tile.bulk_group [%0, {%1, %2, %3}], [%4]; // 1c."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_and_op) {
      asm (
        "cp.reduce.async.bulk.tensor.3d.global.shared::cta.and.tile.bulk_group [%0, {%1, %2, %3}], [%4]; // 1c."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_or_op) {
      asm (
        "cp.reduce.async.bulk.tensor.3d.global.shared::cta.or.tile.bulk_group [%0, {%1, %2, %3}], [%4]; // 1c."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_xor_op) {
      asm (
        "cp.reduce.async.bulk.tensor.3d.global.shared::cta.xor.tile.bulk_group [%0, {%1, %2, %3}], [%4]; // 1c."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.tensor.4d.dst.src.op.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // 1d. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .op        = { .add, .min, .max, .inc, .dec, .and, .or, .xor }
template <cuda_ptx::dot_op Op>
__device__ static inline void cp_reduce_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_t<Op> op,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[4],
  const void* srcMem);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_op Op>
__device__ static inline void cp_reduce_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_t<Op> op,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[4],
  const void* srcMem)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  static_assert(op == op_add || op == op_min || op == op_max || op == op_inc || op == op_dec || op == op_and_op || op == op_or_op || op == op_xor_op, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (op == op_add) {
      asm (
        "cp.reduce.async.bulk.tensor.4d.global.shared::cta.add.tile.bulk_group [%0, {%1, %2, %3, %4}], [%5]; // 1d."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_min) {
      asm (
        "cp.reduce.async.bulk.tensor.4d.global.shared::cta.min.tile.bulk_group [%0, {%1, %2, %3, %4}], [%5]; // 1d."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "cp.reduce.async.bulk.tensor.4d.global.shared::cta.max.tile.bulk_group [%0, {%1, %2, %3, %4}], [%5]; // 1d."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_inc) {
      asm (
        "cp.reduce.async.bulk.tensor.4d.global.shared::cta.inc.tile.bulk_group [%0, {%1, %2, %3, %4}], [%5]; // 1d."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_dec) {
      asm (
        "cp.reduce.async.bulk.tensor.4d.global.shared::cta.dec.tile.bulk_group [%0, {%1, %2, %3, %4}], [%5]; // 1d."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_and_op) {
      asm (
        "cp.reduce.async.bulk.tensor.4d.global.shared::cta.and.tile.bulk_group [%0, {%1, %2, %3, %4}], [%5]; // 1d."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_or_op) {
      asm (
        "cp.reduce.async.bulk.tensor.4d.global.shared::cta.or.tile.bulk_group [%0, {%1, %2, %3, %4}], [%5]; // 1d."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_xor_op) {
      asm (
        "cp.reduce.async.bulk.tensor.4d.global.shared::cta.xor.tile.bulk_group [%0, {%1, %2, %3, %4}], [%5]; // 1d."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// cp.reduce.async.bulk.tensor.5d.dst.src.op.tile.bulk_group [tensorMap, tensorCoords], [srcMem]; // 1e. PTX ISA 80, SM_90
// .dst       = { .global }
// .src       = { .shared::cta }
// .op        = { .add, .min, .max, .inc, .dec, .and, .or, .xor }
template <cuda_ptx::dot_op Op>
__device__ static inline void cp_reduce_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_t<Op> op,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  const void* srcMem);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_op Op>
__device__ static inline void cp_reduce_async_bulk_tensor(
  cuda_ptx::space_global_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::op_t<Op> op,
  const void* tensorMap,
  const std::int32_t (&tensorCoords)[5],
  const void* srcMem)
{
  // space == space_global (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  static_assert(op == op_add || op == op_min || op == op_max || op == op_inc || op == op_dec || op == op_and_op || op == op_or_op || op == op_xor_op, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (op == op_add) {
      asm (
        "cp.reduce.async.bulk.tensor.5d.global.shared::cta.add.tile.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6]; // 1e."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(tensorCoords[4]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_min) {
      asm (
        "cp.reduce.async.bulk.tensor.5d.global.shared::cta.min.tile.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6]; // 1e."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(tensorCoords[4]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "cp.reduce.async.bulk.tensor.5d.global.shared::cta.max.tile.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6]; // 1e."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(tensorCoords[4]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_inc) {
      asm (
        "cp.reduce.async.bulk.tensor.5d.global.shared::cta.inc.tile.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6]; // 1e."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(tensorCoords[4]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_dec) {
      asm (
        "cp.reduce.async.bulk.tensor.5d.global.shared::cta.dec.tile.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6]; // 1e."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(tensorCoords[4]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_and_op) {
      asm (
        "cp.reduce.async.bulk.tensor.5d.global.shared::cta.and.tile.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6]; // 1e."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(tensorCoords[4]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_or_op) {
      asm (
        "cp.reduce.async.bulk.tensor.5d.global.shared::cta.or.tile.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6]; // 1e."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(tensorCoords[4]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    } else if constexpr (op == op_xor_op) {
      asm (
        "cp.reduce.async.bulk.tensor.5d.global.shared::cta.xor.tile.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6]; // 1e."
        :
        : "l"(tensorMap),
          "r"(tensorCoords[0]),
          "r"(tensorCoords[1]),
          "r"(tensorCoords[2]),
          "r"(tensorCoords[3]),
          "r"(tensorCoords[4]),
          "r"(__as_ptr_smem(srcMem))
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_cp_reduce_async_bulk_tensor_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

} // namespace cuda_ptx

#endif // INSTRUCTIONS_CP_REDUCE_ASYNC_BULK_TENSOR_H

// End of ./instructions/cp_reduce_async_bulk_tensor.h
// Start of ./instructions/elect_sync.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_ELECT_SYNC_H
#define INSTRUCTIONS_ELECT_SYNC_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/elect_sync.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// elect.sync _|is_elected, membermask; // PTX ISA 80, SM_90
template <typename = void>
__device__ static inline bool elect_sync(
  const std::uint32_t& membermask);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_elect_sync_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline bool elect_sync(
  const std::uint32_t& membermask)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t is_elected;
    asm volatile (
      "{\n\t"
      ".reg .pred P_OUT; \n\t"
      "elect.sync _|P_OUT, %1; \n\t"
      "selp.b32 %0, 1, 0, P_OUT; \n"
      "}"
      : "=r"(is_elected)
      : "r"(membermask)
      :
    );
    return static_cast<bool>(is_elected);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_elect_sync_is_not_supported_before_SM_90__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 800

} // namespace cuda_ptx

#endif // INSTRUCTIONS_ELECT_SYNC_H

// End of ./instructions/elect_sync.h
// Start of ./instructions/exit.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_EXIT_H
#define INSTRUCTIONS_EXIT_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/exit.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// exit; // PTX ISA 10, SM_50
template <typename = void>
__device__ static inline void exit();
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_exit_is_not_supported_before_SM_50__();
template <typename = void>
__device__ static inline void exit()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    asm volatile (
      "exit;"
      :
      :
      :
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_exit_is_not_supported_before_SM_50__();
  #endif
}
#endif // __libcuda_ptx_isa >= 100

} // namespace cuda_ptx

#endif // INSTRUCTIONS_EXIT_H

// End of ./instructions/exit.h
// Start of ./instructions/fence.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_FENCE_H
#define INSTRUCTIONS_FENCE_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/fence.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// fence.sem.scope; // 1. PTX ISA 60, SM_70
// .sem       = { .sc }
// .scope     = { .cta, .gpu, .sys }
template <cuda_ptx::dot_scope Scope>
__device__ static inline void fence(
  cuda_ptx::sem_sc_t,
  cuda_ptx::scope_t<Scope> scope);
*/
#if __libcuda_ptx_isa >= 600
extern "C" __device__ void __cuda_ptx_fence_is_not_supported_before_SM_70__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline void fence(
  cuda_ptx::sem_sc_t,
  cuda_ptx::scope_t<Scope> scope)
{
  // sem == sem_sc (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_gpu || scope == scope_sys, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    if constexpr (scope == scope_cta) {
      asm volatile (
        "fence.sc.cta; // 1."
        :
        :
        : "memory"
      );
    } else if constexpr (scope == scope_gpu) {
      asm volatile (
        "fence.sc.gpu; // 1."
        :
        :
        : "memory"
      );
    } else if constexpr (scope == scope_sys) {
      asm volatile (
        "fence.sc.sys; // 1."
        :
        :
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_fence_is_not_supported_before_SM_70__();
  #endif
}
#endif // __libcuda_ptx_isa >= 600

/*
// fence.sem.scope; // 2. PTX ISA 78, SM_90
// .sem       = { .sc }
// .scope     = { .cluster }
template <typename = void>
__device__ static inline void fence(
  cuda_ptx::sem_sc_t,
  cuda_ptx::scope_cluster_t);
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_fence_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void fence(
  cuda_ptx::sem_sc_t,
  cuda_ptx::scope_cluster_t)
{
  // sem == sem_sc (due to parameter type constraint)
  // scope == scope_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "fence.sc.cluster; // 2."
      :
      :
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_fence_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// fence.sem.scope; // 1. PTX ISA 60, SM_70
// .sem       = { .acq_rel }
// .scope     = { .cta, .gpu, .sys }
template <cuda_ptx::dot_scope Scope>
__device__ static inline void fence(
  cuda_ptx::sem_acq_rel_t,
  cuda_ptx::scope_t<Scope> scope);
*/
#if __libcuda_ptx_isa >= 600
extern "C" __device__ void __cuda_ptx_fence_is_not_supported_before_SM_70__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline void fence(
  cuda_ptx::sem_acq_rel_t,
  cuda_ptx::scope_t<Scope> scope)
{
  // sem == sem_acq_rel (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_gpu || scope == scope_sys, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    if constexpr (scope == scope_cta) {
      asm volatile (
        "fence.acq_rel.cta; // 1."
        :
        :
        : "memory"
      );
    } else if constexpr (scope == scope_gpu) {
      asm volatile (
        "fence.acq_rel.gpu; // 1."
        :
        :
        : "memory"
      );
    } else if constexpr (scope == scope_sys) {
      asm volatile (
        "fence.acq_rel.sys; // 1."
        :
        :
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_fence_is_not_supported_before_SM_70__();
  #endif
}
#endif // __libcuda_ptx_isa >= 600

/*
// fence.sem.scope; // 2. PTX ISA 78, SM_90
// .sem       = { .acq_rel }
// .scope     = { .cluster }
template <typename = void>
__device__ static inline void fence(
  cuda_ptx::sem_acq_rel_t,
  cuda_ptx::scope_cluster_t);
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_fence_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void fence(
  cuda_ptx::sem_acq_rel_t,
  cuda_ptx::scope_cluster_t)
{
  // sem == sem_acq_rel (due to parameter type constraint)
  // scope == scope_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "fence.acq_rel.cluster; // 2."
      :
      :
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_fence_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// fence.sem.scope; // PTX ISA 86, SM_90
// .sem       = { .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
template <cuda_ptx::dot_scope Scope>
__device__ static inline void fence(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::scope_t<Scope> scope);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_fence_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline void fence(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::scope_t<Scope> scope)
{
  // sem == sem_acquire (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (scope == scope_cta) {
      asm volatile (
        "fence.acquire.cta;"
        :
        :
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm volatile (
        "fence.acquire.cluster;"
        :
        :
        : "memory"
      );
    } else if constexpr (scope == scope_gpu) {
      asm volatile (
        "fence.acquire.gpu;"
        :
        :
        : "memory"
      );
    } else if constexpr (scope == scope_sys) {
      asm volatile (
        "fence.acquire.sys;"
        :
        :
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_fence_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// fence.sem.scope; // PTX ISA 86, SM_90
// .sem       = { .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
template <cuda_ptx::dot_scope Scope>
__device__ static inline void fence(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_t<Scope> scope);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_fence_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline void fence(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_t<Scope> scope)
{
  // sem == sem_release (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (scope == scope_cta) {
      asm volatile (
        "fence.release.cta;"
        :
        :
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm volatile (
        "fence.release.cluster;"
        :
        :
        : "memory"
      );
    } else if constexpr (scope == scope_gpu) {
      asm volatile (
        "fence.release.gpu;"
        :
        :
        : "memory"
      );
    } else if constexpr (scope == scope_sys) {
      asm volatile (
        "fence.release.sys;"
        :
        :
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_fence_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860
/*
// fence.mbarrier_init.sem.scope; // 3. PTX ISA 80, SM_90
// .sem       = { .release }
// .scope     = { .cluster }
template <typename = void>
__device__ static inline void fence_mbarrier_init(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_cluster_t);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_fence_mbarrier_init_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void fence_mbarrier_init(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_cluster_t)
{
  // sem == sem_release (due to parameter type constraint)
  // scope == scope_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "fence.mbarrier_init.release.cluster; // 3."
      :
      :
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_fence_mbarrier_init_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800
/*
// fence.proxy.alias; // 4. PTX ISA 75, SM_70
template <typename = void>
__device__ static inline void fence_proxy_alias();
*/
#if __libcuda_ptx_isa >= 750
extern "C" __device__ void __cuda_ptx_fence_proxy_alias_is_not_supported_before_SM_70__();
template <typename = void>
__device__ static inline void fence_proxy_alias()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    asm volatile (
      "fence.proxy.alias; // 4."
      :
      :
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_fence_proxy_alias_is_not_supported_before_SM_70__();
  #endif
}
#endif // __libcuda_ptx_isa >= 750
/*
// fence.proxy.async; // 5. PTX ISA 80, SM_90
template <typename = void>
__device__ static inline void fence_proxy_async();
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_fence_proxy_async_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void fence_proxy_async()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "fence.proxy.async; // 5."
      :
      :
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_fence_proxy_async_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// fence.proxy.async.space; // 6. PTX ISA 80, SM_90
// .space     = { .global, .shared::cluster, .shared::cta }
template <cuda_ptx::dot_space Space>
__device__ static inline void fence_proxy_async(
  cuda_ptx::space_t<Space> space);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_fence_proxy_async_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_space Space>
__device__ static inline void fence_proxy_async(
  cuda_ptx::space_t<Space> space)
{
  static_assert(space == space_global || space == space_cluster || space == space_shared, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (space == space_global) {
      asm volatile (
        "fence.proxy.async.global; // 6."
        :
        :
        : "memory"
      );
    } else if constexpr (space == space_cluster) {
      asm volatile (
        "fence.proxy.async.shared::cluster; // 6."
        :
        :
        : "memory"
      );
    } else if constexpr (space == space_shared) {
      asm volatile (
        "fence.proxy.async.shared::cta; // 6."
        :
        :
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_fence_proxy_async_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800
/*
// fence.proxy.tensormap::generic.release.scope; // 7. PTX ISA 83, SM_90
// .sem       = { .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
template <cuda_ptx::dot_scope Scope>
__device__ static inline void fence_proxy_tensormap_generic(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_t<Scope> scope);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_fence_proxy_tensormap_generic_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline void fence_proxy_tensormap_generic(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_t<Scope> scope)
{
  // sem == sem_release (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (scope == scope_cta) {
      asm volatile (
        "fence.proxy.tensormap::generic.release.cta; // 7."
        :
        :
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm volatile (
        "fence.proxy.tensormap::generic.release.cluster; // 7."
        :
        :
        : "memory"
      );
    } else if constexpr (scope == scope_gpu) {
      asm volatile (
        "fence.proxy.tensormap::generic.release.gpu; // 7."
        :
        :
        : "memory"
      );
    } else if constexpr (scope == scope_sys) {
      asm volatile (
        "fence.proxy.tensormap::generic.release.sys; // 7."
        :
        :
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_fence_proxy_tensormap_generic_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// fence.proxy.tensormap::generic.sem.scope [addr], size; // 8. PTX ISA 83, SM_90
// .sem       = { .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
template <int N32, cuda_ptx::dot_scope Scope>
__device__ static inline void fence_proxy_tensormap_generic(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::scope_t<Scope> scope,
  const void* addr,
  cuda_ptx::n32_t<N32> size);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_fence_proxy_tensormap_generic_is_not_supported_before_SM_90__();
template <int N32, cuda_ptx::dot_scope Scope>
__device__ static inline void fence_proxy_tensormap_generic(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::scope_t<Scope> scope,
  const void* addr,
  cuda_ptx::n32_t<N32> size)
{
  // sem == sem_acquire (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (scope == scope_cta) {
      asm volatile (
        "fence.proxy.tensormap::generic.acquire.cta [%0], %1; // 8."
        :
        : "l"(addr),
          "n"(size.value)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm volatile (
        "fence.proxy.tensormap::generic.acquire.cluster [%0], %1; // 8."
        :
        : "l"(addr),
          "n"(size.value)
        : "memory"
      );
    } else if constexpr (scope == scope_gpu) {
      asm volatile (
        "fence.proxy.tensormap::generic.acquire.gpu [%0], %1; // 8."
        :
        : "l"(addr),
          "n"(size.value)
        : "memory"
      );
    } else if constexpr (scope == scope_sys) {
      asm volatile (
        "fence.proxy.tensormap::generic.acquire.sys [%0], %1; // 8."
        :
        : "l"(addr),
          "n"(size.value)
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_fence_proxy_tensormap_generic_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830
/*
// fence.proxy.async::generic.sem.sync_restrict::space.scope; // PTX ISA 86, SM_90
// .sem       = { .acquire }
// .space     = { .shared::cluster }
// .scope     = { .cluster }
template <typename = void>
__device__ static inline void fence_proxy_async_generic_sync_restrict(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::space_cluster_t,
  cuda_ptx::scope_cluster_t);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_fence_proxy_async_generic_sync_restrict_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void fence_proxy_async_generic_sync_restrict(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::space_cluster_t,
  cuda_ptx::scope_cluster_t)
{
  // sem == sem_acquire (due to parameter type constraint)
  // space == space_cluster (due to parameter type constraint)
  // scope == scope_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "fence.proxy.async::generic.acquire.sync_restrict::shared::cluster.cluster;"
      :
      :
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_fence_proxy_async_generic_sync_restrict_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// fence.proxy.async::generic.sem.sync_restrict::space.scope; // PTX ISA 86, SM_90
// .sem       = { .release }
// .space     = { .shared::cta }
// .scope     = { .cluster }
template <typename = void>
__device__ static inline void fence_proxy_async_generic_sync_restrict(
  cuda_ptx::sem_release_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::scope_cluster_t);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_fence_proxy_async_generic_sync_restrict_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void fence_proxy_async_generic_sync_restrict(
  cuda_ptx::sem_release_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::scope_cluster_t)
{
  // sem == sem_release (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // scope == scope_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "fence.proxy.async::generic.release.sync_restrict::shared::cta.cluster;"
      :
      :
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_fence_proxy_async_generic_sync_restrict_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860
/*
// fence.sem.sync_restrict::space.scope; // PTX ISA 86, SM_90
// .sem       = { .acquire }
// .space     = { .shared::cluster }
// .scope     = { .cluster }
template <typename = void>
__device__ static inline void fence_sync_restrict(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::space_cluster_t,
  cuda_ptx::scope_cluster_t);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_fence_sync_restrict_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void fence_sync_restrict(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::space_cluster_t,
  cuda_ptx::scope_cluster_t)
{
  // sem == sem_acquire (due to parameter type constraint)
  // space == space_cluster (due to parameter type constraint)
  // scope == scope_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "fence.acquire.sync_restrict::shared::cluster.cluster;"
      :
      :
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_fence_sync_restrict_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// fence.sem.sync_restrict::space.scope; // PTX ISA 86, SM_90
// .sem       = { .release }
// .space     = { .shared::cta }
// .scope     = { .cluster }
template <typename = void>
__device__ static inline void fence_sync_restrict(
  cuda_ptx::sem_release_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::scope_cluster_t);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_fence_sync_restrict_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void fence_sync_restrict(
  cuda_ptx::sem_release_t,
  cuda_ptx::space_shared_t,
  cuda_ptx::scope_cluster_t)
{
  // sem == sem_release (due to parameter type constraint)
  // space == space_shared (due to parameter type constraint)
  // scope == scope_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "fence.release.sync_restrict::shared::cta.cluster;"
      :
      :
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_fence_sync_restrict_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

} // namespace cuda_ptx

#endif // INSTRUCTIONS_FENCE_H

// End of ./instructions/fence.h
// Start of ./instructions/get_sreg.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_GET_SREG_H
#define INSTRUCTIONS_GET_SREG_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/get_sreg.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// mov.u32 sreg_value, %%tid.x; // PTX ISA 20
template <typename = void>
__device__ static inline std::uint32_t get_sreg_tid_x();
*/
#if __libcuda_ptx_isa >= 200
template <typename = void>
__device__ static inline std::uint32_t get_sreg_tid_x()
{
  std::uint32_t sreg_value;
  asm (
    "mov.u32 %0, %%tid.x;"
    : "=r"(sreg_value)
    :
    :
  );
  return sreg_value;
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u32 sreg_value, %%tid.y; // PTX ISA 20
template <typename = void>
__device__ static inline std::uint32_t get_sreg_tid_y();
*/
#if __libcuda_ptx_isa >= 200
template <typename = void>
__device__ static inline std::uint32_t get_sreg_tid_y()
{
  std::uint32_t sreg_value;
  asm (
    "mov.u32 %0, %%tid.y;"
    : "=r"(sreg_value)
    :
    :
  );
  return sreg_value;
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u32 sreg_value, %%tid.z; // PTX ISA 20
template <typename = void>
__device__ static inline std::uint32_t get_sreg_tid_z();
*/
#if __libcuda_ptx_isa >= 200
template <typename = void>
__device__ static inline std::uint32_t get_sreg_tid_z()
{
  std::uint32_t sreg_value;
  asm (
    "mov.u32 %0, %%tid.z;"
    : "=r"(sreg_value)
    :
    :
  );
  return sreg_value;
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u32 sreg_value, %%ntid.x; // PTX ISA 20
template <typename = void>
__device__ static inline std::uint32_t get_sreg_ntid_x();
*/
#if __libcuda_ptx_isa >= 200
template <typename = void>
__device__ static inline std::uint32_t get_sreg_ntid_x()
{
  std::uint32_t sreg_value;
  asm volatile (
    "mov.u32 %0, %%ntid.x;"
    : "=r"(sreg_value)
    :
    :
  );
  return sreg_value;
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u32 sreg_value, %%ntid.y; // PTX ISA 20
template <typename = void>
__device__ static inline std::uint32_t get_sreg_ntid_y();
*/
#if __libcuda_ptx_isa >= 200
template <typename = void>
__device__ static inline std::uint32_t get_sreg_ntid_y()
{
  std::uint32_t sreg_value;
  asm volatile (
    "mov.u32 %0, %%ntid.y;"
    : "=r"(sreg_value)
    :
    :
  );
  return sreg_value;
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u32 sreg_value, %%ntid.z; // PTX ISA 20
template <typename = void>
__device__ static inline std::uint32_t get_sreg_ntid_z();
*/
#if __libcuda_ptx_isa >= 200
template <typename = void>
__device__ static inline std::uint32_t get_sreg_ntid_z()
{
  std::uint32_t sreg_value;
  asm volatile (
    "mov.u32 %0, %%ntid.z;"
    : "=r"(sreg_value)
    :
    :
  );
  return sreg_value;
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u32 sreg_value, %%laneid; // PTX ISA 13
template <typename = void>
__device__ static inline std::uint32_t get_sreg_laneid();
*/
#if __libcuda_ptx_isa >= 130
template <typename = void>
__device__ static inline std::uint32_t get_sreg_laneid()
{
  std::uint32_t sreg_value;
  asm (
    "mov.u32 %0, %%laneid;"
    : "=r"(sreg_value)
    :
    :
  );
  return sreg_value;
}
#endif // __libcuda_ptx_isa >= 130

/*
// mov.u32 sreg_value, %%warpid; // PTX ISA 13
template <typename = void>
__device__ static inline std::uint32_t get_sreg_warpid();
*/
#if __libcuda_ptx_isa >= 130
template <typename = void>
__device__ static inline std::uint32_t get_sreg_warpid()
{
  std::uint32_t sreg_value;
  asm volatile (
    "mov.u32 %0, %%warpid;"
    : "=r"(sreg_value)
    :
    :
  );
  return sreg_value;
}
#endif // __libcuda_ptx_isa >= 130

/*
// mov.u32 sreg_value, %%nwarpid; // PTX ISA 20, SM_35
template <typename = void>
__device__ static inline std::uint32_t get_sreg_nwarpid();
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_get_sreg_nwarpid_is_not_supported_before_SM_35__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_nwarpid()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 350
    std::uint32_t sreg_value;
    asm volatile (
      "mov.u32 %0, %%nwarpid;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_nwarpid_is_not_supported_before_SM_35__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u32 sreg_value, %%ctaid.x; // PTX ISA 20
template <typename = void>
__device__ static inline std::uint32_t get_sreg_ctaid_x();
*/
#if __libcuda_ptx_isa >= 200
template <typename = void>
__device__ static inline std::uint32_t get_sreg_ctaid_x()
{
  std::uint32_t sreg_value;
  asm (
    "mov.u32 %0, %%ctaid.x;"
    : "=r"(sreg_value)
    :
    :
  );
  return sreg_value;
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u32 sreg_value, %%ctaid.y; // PTX ISA 20
template <typename = void>
__device__ static inline std::uint32_t get_sreg_ctaid_y();
*/
#if __libcuda_ptx_isa >= 200
template <typename = void>
__device__ static inline std::uint32_t get_sreg_ctaid_y()
{
  std::uint32_t sreg_value;
  asm (
    "mov.u32 %0, %%ctaid.y;"
    : "=r"(sreg_value)
    :
    :
  );
  return sreg_value;
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u32 sreg_value, %%ctaid.z; // PTX ISA 20
template <typename = void>
__device__ static inline std::uint32_t get_sreg_ctaid_z();
*/
#if __libcuda_ptx_isa >= 200
template <typename = void>
__device__ static inline std::uint32_t get_sreg_ctaid_z()
{
  std::uint32_t sreg_value;
  asm (
    "mov.u32 %0, %%ctaid.z;"
    : "=r"(sreg_value)
    :
    :
  );
  return sreg_value;
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u32 sreg_value, %%nctaid.x; // PTX ISA 20
template <typename = void>
__device__ static inline std::uint32_t get_sreg_nctaid_x();
*/
#if __libcuda_ptx_isa >= 200
template <typename = void>
__device__ static inline std::uint32_t get_sreg_nctaid_x()
{
  std::uint32_t sreg_value;
  asm (
    "mov.u32 %0, %%nctaid.x;"
    : "=r"(sreg_value)
    :
    :
  );
  return sreg_value;
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u32 sreg_value, %%nctaid.y; // PTX ISA 20
template <typename = void>
__device__ static inline std::uint32_t get_sreg_nctaid_y();
*/
#if __libcuda_ptx_isa >= 200
template <typename = void>
__device__ static inline std::uint32_t get_sreg_nctaid_y()
{
  std::uint32_t sreg_value;
  asm (
    "mov.u32 %0, %%nctaid.y;"
    : "=r"(sreg_value)
    :
    :
  );
  return sreg_value;
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u32 sreg_value, %%nctaid.z; // PTX ISA 20
template <typename = void>
__device__ static inline std::uint32_t get_sreg_nctaid_z();
*/
#if __libcuda_ptx_isa >= 200
template <typename = void>
__device__ static inline std::uint32_t get_sreg_nctaid_z()
{
  std::uint32_t sreg_value;
  asm (
    "mov.u32 %0, %%nctaid.z;"
    : "=r"(sreg_value)
    :
    :
  );
  return sreg_value;
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u32 sreg_value, %%smid; // PTX ISA 13
template <typename = void>
__device__ static inline std::uint32_t get_sreg_smid();
*/
#if __libcuda_ptx_isa >= 130
template <typename = void>
__device__ static inline std::uint32_t get_sreg_smid()
{
  std::uint32_t sreg_value;
  asm (
    "mov.u32 %0, %%smid;"
    : "=r"(sreg_value)
    :
    :
  );
  return sreg_value;
}
#endif // __libcuda_ptx_isa >= 130

/*
// mov.u32 sreg_value, %%nsmid; // PTX ISA 20, SM_35
template <typename = void>
__device__ static inline std::uint32_t get_sreg_nsmid();
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_get_sreg_nsmid_is_not_supported_before_SM_35__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_nsmid()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 350
    std::uint32_t sreg_value;
    asm volatile (
      "mov.u32 %0, %%nsmid;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_nsmid_is_not_supported_before_SM_35__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u64 sreg_value, %%gridid; // PTX ISA 30
template <typename = void>
__device__ static inline std::uint64_t get_sreg_gridid();
*/
#if __libcuda_ptx_isa >= 300
template <typename = void>
__device__ static inline std::uint64_t get_sreg_gridid()
{
  std::uint64_t sreg_value;
  asm (
    "mov.u64 %0, %%gridid;"
    : "=l"(sreg_value)
    :
    :
  );
  return sreg_value;
}
#endif // __libcuda_ptx_isa >= 300

/*
// mov.pred sreg_value, %%is_explicit_cluster; // PTX ISA 78, SM_90
template <typename = void>
__device__ static inline bool get_sreg_is_explicit_cluster();
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_get_sreg_is_explicit_cluster_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline bool get_sreg_is_explicit_cluster()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t sreg_value;
    asm (
      "{\n\t"
      ".reg .pred P_OUT; \n\t"
      "mov.pred P_OUT, %%is_explicit_cluster; \n\t"
      "selp.b32 %0, 1, 0, P_OUT; \n"
      "}"
      : "=r"(sreg_value)
      :
      :
    );
    return static_cast<bool>(sreg_value);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_is_explicit_cluster_is_not_supported_before_SM_90__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mov.u32 sreg_value, %%clusterid.x; // PTX ISA 78, SM_90
template <typename = void>
__device__ static inline std::uint32_t get_sreg_clusterid_x();
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_get_sreg_clusterid_x_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_clusterid_x()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%clusterid.x;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_clusterid_x_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mov.u32 sreg_value, %%clusterid.y; // PTX ISA 78, SM_90
template <typename = void>
__device__ static inline std::uint32_t get_sreg_clusterid_y();
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_get_sreg_clusterid_y_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_clusterid_y()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%clusterid.y;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_clusterid_y_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mov.u32 sreg_value, %%clusterid.z; // PTX ISA 78, SM_90
template <typename = void>
__device__ static inline std::uint32_t get_sreg_clusterid_z();
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_get_sreg_clusterid_z_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_clusterid_z()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%clusterid.z;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_clusterid_z_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mov.u32 sreg_value, %%nclusterid.x; // PTX ISA 78, SM_90
template <typename = void>
__device__ static inline std::uint32_t get_sreg_nclusterid_x();
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_get_sreg_nclusterid_x_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_nclusterid_x()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%nclusterid.x;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_nclusterid_x_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mov.u32 sreg_value, %%nclusterid.y; // PTX ISA 78, SM_90
template <typename = void>
__device__ static inline std::uint32_t get_sreg_nclusterid_y();
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_get_sreg_nclusterid_y_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_nclusterid_y()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%nclusterid.y;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_nclusterid_y_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mov.u32 sreg_value, %%nclusterid.z; // PTX ISA 78, SM_90
template <typename = void>
__device__ static inline std::uint32_t get_sreg_nclusterid_z();
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_get_sreg_nclusterid_z_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_nclusterid_z()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%nclusterid.z;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_nclusterid_z_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mov.u32 sreg_value, %%cluster_ctaid.x; // PTX ISA 78, SM_90
template <typename = void>
__device__ static inline std::uint32_t get_sreg_cluster_ctaid_x();
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_get_sreg_cluster_ctaid_x_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_cluster_ctaid_x()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%cluster_ctaid.x;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_cluster_ctaid_x_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mov.u32 sreg_value, %%cluster_ctaid.y; // PTX ISA 78, SM_90
template <typename = void>
__device__ static inline std::uint32_t get_sreg_cluster_ctaid_y();
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_get_sreg_cluster_ctaid_y_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_cluster_ctaid_y()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%cluster_ctaid.y;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_cluster_ctaid_y_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mov.u32 sreg_value, %%cluster_ctaid.z; // PTX ISA 78, SM_90
template <typename = void>
__device__ static inline std::uint32_t get_sreg_cluster_ctaid_z();
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_get_sreg_cluster_ctaid_z_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_cluster_ctaid_z()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%cluster_ctaid.z;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_cluster_ctaid_z_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mov.u32 sreg_value, %%cluster_nctaid.x; // PTX ISA 78, SM_90
template <typename = void>
__device__ static inline std::uint32_t get_sreg_cluster_nctaid_x();
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_get_sreg_cluster_nctaid_x_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_cluster_nctaid_x()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%cluster_nctaid.x;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_cluster_nctaid_x_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mov.u32 sreg_value, %%cluster_nctaid.y; // PTX ISA 78, SM_90
template <typename = void>
__device__ static inline std::uint32_t get_sreg_cluster_nctaid_y();
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_get_sreg_cluster_nctaid_y_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_cluster_nctaid_y()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%cluster_nctaid.y;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_cluster_nctaid_y_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mov.u32 sreg_value, %%cluster_nctaid.z; // PTX ISA 78, SM_90
template <typename = void>
__device__ static inline std::uint32_t get_sreg_cluster_nctaid_z();
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_get_sreg_cluster_nctaid_z_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_cluster_nctaid_z()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%cluster_nctaid.z;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_cluster_nctaid_z_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mov.u32 sreg_value, %%cluster_ctarank; // PTX ISA 78, SM_90
template <typename = void>
__device__ static inline std::uint32_t get_sreg_cluster_ctarank();
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_get_sreg_cluster_ctarank_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_cluster_ctarank()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%cluster_ctarank;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_cluster_ctarank_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mov.u32 sreg_value, %%cluster_nctarank; // PTX ISA 78, SM_90
template <typename = void>
__device__ static inline std::uint32_t get_sreg_cluster_nctarank();
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_get_sreg_cluster_nctarank_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_cluster_nctarank()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%cluster_nctarank;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_cluster_nctarank_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mov.u32 sreg_value, %%lanemask_eq; // PTX ISA 20, SM_35
template <typename = void>
__device__ static inline std::uint32_t get_sreg_lanemask_eq();
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_get_sreg_lanemask_eq_is_not_supported_before_SM_35__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_lanemask_eq()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 350
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%lanemask_eq;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_lanemask_eq_is_not_supported_before_SM_35__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u32 sreg_value, %%lanemask_le; // PTX ISA 20, SM_35
template <typename = void>
__device__ static inline std::uint32_t get_sreg_lanemask_le();
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_get_sreg_lanemask_le_is_not_supported_before_SM_35__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_lanemask_le()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 350
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%lanemask_le;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_lanemask_le_is_not_supported_before_SM_35__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u32 sreg_value, %%lanemask_lt; // PTX ISA 20, SM_35
template <typename = void>
__device__ static inline std::uint32_t get_sreg_lanemask_lt();
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_get_sreg_lanemask_lt_is_not_supported_before_SM_35__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_lanemask_lt()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 350
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%lanemask_lt;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_lanemask_lt_is_not_supported_before_SM_35__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u32 sreg_value, %%lanemask_ge; // PTX ISA 20, SM_35
template <typename = void>
__device__ static inline std::uint32_t get_sreg_lanemask_ge();
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_get_sreg_lanemask_ge_is_not_supported_before_SM_35__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_lanemask_ge()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 350
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%lanemask_ge;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_lanemask_ge_is_not_supported_before_SM_35__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u32 sreg_value, %%lanemask_gt; // PTX ISA 20, SM_35
template <typename = void>
__device__ static inline std::uint32_t get_sreg_lanemask_gt();
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_get_sreg_lanemask_gt_is_not_supported_before_SM_35__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_lanemask_gt()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 350
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%lanemask_gt;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_lanemask_gt_is_not_supported_before_SM_35__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u32 sreg_value, %%clock; // PTX ISA 10
template <typename = void>
__device__ static inline std::uint32_t get_sreg_clock();
*/
#if __libcuda_ptx_isa >= 100
template <typename = void>
__device__ static inline std::uint32_t get_sreg_clock()
{
  std::uint32_t sreg_value;
  asm volatile (
    "mov.u32 %0, %%clock;"
    : "=r"(sreg_value)
    :
    :
  );
  return sreg_value;
}
#endif // __libcuda_ptx_isa >= 100

/*
// mov.u32 sreg_value, %%clock_hi; // PTX ISA 50, SM_35
template <typename = void>
__device__ static inline std::uint32_t get_sreg_clock_hi();
*/
#if __libcuda_ptx_isa >= 500
extern "C" __device__ void __cuda_ptx_get_sreg_clock_hi_is_not_supported_before_SM_35__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_clock_hi()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 350
    std::uint32_t sreg_value;
    asm volatile (
      "mov.u32 %0, %%clock_hi;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_clock_hi_is_not_supported_before_SM_35__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 500

/*
// mov.u64 sreg_value, %%clock64; // PTX ISA 20, SM_35
template <typename = void>
__device__ static inline std::uint64_t get_sreg_clock64();
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_get_sreg_clock64_is_not_supported_before_SM_35__();
template <typename = void>
__device__ static inline std::uint64_t get_sreg_clock64()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 350
    std::uint64_t sreg_value;
    asm volatile (
      "mov.u64 %0, %%clock64;"
      : "=l"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_clock64_is_not_supported_before_SM_35__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// mov.u64 sreg_value, %%globaltimer; // PTX ISA 31, SM_35
template <typename = void>
__device__ static inline std::uint64_t get_sreg_globaltimer();
*/
#if __libcuda_ptx_isa >= 310
extern "C" __device__ void __cuda_ptx_get_sreg_globaltimer_is_not_supported_before_SM_35__();
template <typename = void>
__device__ static inline std::uint64_t get_sreg_globaltimer()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 350
    std::uint64_t sreg_value;
    asm volatile (
      "mov.u64 %0, %%globaltimer;"
      : "=l"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_globaltimer_is_not_supported_before_SM_35__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 310

/*
// mov.u32 sreg_value, %%globaltimer_lo; // PTX ISA 31, SM_35
template <typename = void>
__device__ static inline std::uint32_t get_sreg_globaltimer_lo();
*/
#if __libcuda_ptx_isa >= 310
extern "C" __device__ void __cuda_ptx_get_sreg_globaltimer_lo_is_not_supported_before_SM_35__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_globaltimer_lo()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 350
    std::uint32_t sreg_value;
    asm volatile (
      "mov.u32 %0, %%globaltimer_lo;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_globaltimer_lo_is_not_supported_before_SM_35__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 310

/*
// mov.u32 sreg_value, %%globaltimer_hi; // PTX ISA 31, SM_35
template <typename = void>
__device__ static inline std::uint32_t get_sreg_globaltimer_hi();
*/
#if __libcuda_ptx_isa >= 310
extern "C" __device__ void __cuda_ptx_get_sreg_globaltimer_hi_is_not_supported_before_SM_35__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_globaltimer_hi()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 350
    std::uint32_t sreg_value;
    asm volatile (
      "mov.u32 %0, %%globaltimer_hi;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_globaltimer_hi_is_not_supported_before_SM_35__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 310

/*
// mov.u32 sreg_value, %%total_smem_size; // PTX ISA 41, SM_35
template <typename = void>
__device__ static inline std::uint32_t get_sreg_total_smem_size();
*/
#if __libcuda_ptx_isa >= 410
extern "C" __device__ void __cuda_ptx_get_sreg_total_smem_size_is_not_supported_before_SM_35__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_total_smem_size()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 350
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%total_smem_size;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_total_smem_size_is_not_supported_before_SM_35__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 410

/*
// mov.u32 sreg_value, %%aggr_smem_size; // PTX ISA 81, SM_90
template <typename = void>
__device__ static inline std::uint32_t get_sreg_aggr_smem_size();
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_get_sreg_aggr_smem_size_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_aggr_smem_size()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%aggr_smem_size;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_aggr_smem_size_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// mov.u32 sreg_value, %%dynamic_smem_size; // PTX ISA 41, SM_35
template <typename = void>
__device__ static inline std::uint32_t get_sreg_dynamic_smem_size();
*/
#if __libcuda_ptx_isa >= 410
extern "C" __device__ void __cuda_ptx_get_sreg_dynamic_smem_size_is_not_supported_before_SM_35__();
template <typename = void>
__device__ static inline std::uint32_t get_sreg_dynamic_smem_size()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 350
    std::uint32_t sreg_value;
    asm (
      "mov.u32 %0, %%dynamic_smem_size;"
      : "=r"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_dynamic_smem_size_is_not_supported_before_SM_35__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 410

/*
// mov.u64 sreg_value, %%current_graph_exec; // PTX ISA 80, SM_50
template <typename = void>
__device__ static inline std::uint64_t get_sreg_current_graph_exec();
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_get_sreg_current_graph_exec_is_not_supported_before_SM_50__();
template <typename = void>
__device__ static inline std::uint64_t get_sreg_current_graph_exec()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint64_t sreg_value;
    asm (
      "mov.u64 %0, %%current_graph_exec;"
      : "=l"(sreg_value)
      :
      :
    );
    return sreg_value;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_get_sreg_current_graph_exec_is_not_supported_before_SM_50__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 800

} // namespace cuda_ptx

#endif // INSTRUCTIONS_GET_SREG_H

// End of ./instructions/get_sreg.h
// Start of ./instructions/getctarank.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_GETCTARANK_H
#define INSTRUCTIONS_GETCTARANK_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/getctarank.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// getctarank.space.u32 dest, addr; // PTX ISA 78, SM_90
// .space     = { .shared::cluster }
template <typename = void>
__device__ static inline std::uint32_t getctarank(
  cuda_ptx::space_cluster_t,
  const void* addr);
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_getctarank_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint32_t getctarank(
  cuda_ptx::space_cluster_t,
  const void* addr)
{
  // space == space_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t dest;
    asm (
      "getctarank.shared::cluster.u32 %0, %1;"
      : "=r"(dest)
      : "r"(__as_ptr_smem(addr))
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_getctarank_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

} // namespace cuda_ptx

#endif // INSTRUCTIONS_GETCTARANK_H

// End of ./instructions/getctarank.h
// Start of ./instructions/ld.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_LD_H
#define INSTRUCTIONS_LD_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/ld.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// ld.space.b8 dest, [addr]; // PTX ISA 10, SM_50
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld(
  cuda_ptx::space_global_t,
  const B8* addr);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_ld_is_not_supported_before_SM_50__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld(
  cuda_ptx::space_global_t,
  const B8* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "ld.global.b8 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_is_not_supported_before_SM_50__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 100

/*
// ld.space.L2::256B.b8 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L2_256B_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L2::256B.b8 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.b16 dest, [addr]; // PTX ISA 10, SM_50
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld(
  cuda_ptx::space_global_t,
  const B16* addr);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_ld_is_not_supported_before_SM_50__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld(
  cuda_ptx::space_global_t,
  const B16* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint16_t dest;
    asm (
      "ld.global.b16 %0, [%1];"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_is_not_supported_before_SM_50__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 100

/*
// ld.space.L2::256B.b16 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L2_256B_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.L2::256B.b16 %0, [%1];"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L2_256B_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.b32 dest, [addr]; // PTX ISA 10, SM_50
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld(
  cuda_ptx::space_global_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_ld_is_not_supported_before_SM_50__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld(
  cuda_ptx::space_global_t,
  const B32* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "ld.global.b32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_is_not_supported_before_SM_50__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 100

/*
// ld.space.L2::256B.b32 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L2_256B_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L2::256B.b32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.b64 dest, [addr]; // PTX ISA 10, SM_50
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld(
  cuda_ptx::space_global_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_ld_is_not_supported_before_SM_50__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld(
  cuda_ptx::space_global_t,
  const B64* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint64_t dest;
    asm (
      "ld.global.b64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_is_not_supported_before_SM_50__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 100

/*
// ld.space.L2::256B.b64 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L2_256B_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.L2::256B.b64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L2_256B_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.b128 dest, [addr]; // PTX ISA 83, SM_70
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld(
  cuda_ptx::space_global_t,
  const B128* addr);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_is_not_supported_before_SM_70__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld(
  cuda_ptx::space_global_t,
  const B128* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.b128 B128_dest, [%2]; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_is_not_supported_before_SM_70__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.L2::256B.b128 dest, [addr]; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_L2_256B_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.L2::256B.b128 B128_dest, [%2]; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L2_256B_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.v4.b64 dest, [addr]; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld(
  cuda_ptx::space_global_t,
  const B256* addr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_ld_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld(
  cuda_ptx::space_global_t,
  const B256* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    cuda_ptx::longlong4_32a dest;
    asm (
      "ld.global.v4.b64 {%0, %1, %2, %3}, [%4];"
      : "=l"(dest.x),
        "=l"(dest.y),
        "=l"(dest.z),
        "=l"(dest.w)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B256*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_is_not_supported_before_SM_100__();
    cuda_ptx::longlong4_32a err_out_var{0, 0, 0, 0};
    return *reinterpret_cast<B256*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// ld.space.L2::cache_hint.b8 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L2::cache_hint.b8 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L2::cache_hint.L2::256B.b8 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L2::cache_hint.L2::256B.b8 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L2::cache_hint.b16 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.L2::cache_hint.b16 %0, [%1], %2;"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L2::cache_hint.L2::256B.b16 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.L2::cache_hint.L2::256B.b16 %0, [%1], %2;"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L2::cache_hint.b32 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L2::cache_hint.b32 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L2::cache_hint.L2::256B.b32 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L2::cache_hint.L2::256B.b32 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L2::cache_hint.b64 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.L2::cache_hint.b64 %0, [%1], %2;"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L2::cache_hint.L2::256B.b64 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.L2::cache_hint.L2::256B.b64 %0, [%1], %2;"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L2::cache_hint.b128 dest, [addr], cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.L2::cache_hint.b128 B128_dest, [%2], %3; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L2_cache_hint_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.L2::cache_hint.L2::256B.b128 dest, [addr], cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.L2::cache_hint.L2::256B.b128 B128_dest, [%2], %3; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.L2::cache_hint.v4.b64 dest, [addr], cache_policy; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B256* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_ld_L2_cache_hint_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B256* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    cuda_ptx::longlong4_32a dest;
    asm (
      "ld.global.L2::cache_hint.v4.b64 {%0, %1, %2, %3}, [%4], %5;"
      : "=l"(dest.x),
        "=l"(dest.y),
        "=l"(dest.z),
        "=l"(dest.w)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B256*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L2_cache_hint_is_not_supported_before_SM_100__();
    cuda_ptx::longlong4_32a err_out_var{0, 0, 0, 0};
    return *reinterpret_cast<B256*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// ld.space.L1::evict_first.b8 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_evict_first(
  cuda_ptx::space_global_t,
  const B8* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_is_not_supported_before_SM_70__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_evict_first(
  cuda_ptx::space_global_t,
  const B8* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint32_t dest;
    asm (
      "ld.global.L1::evict_first.b8 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_is_not_supported_before_SM_70__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_first.L2::256B.b8 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L1::evict_first.L2::256B.b8 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_first.b16 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_evict_first(
  cuda_ptx::space_global_t,
  const B16* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_is_not_supported_before_SM_70__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_evict_first(
  cuda_ptx::space_global_t,
  const B16* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint16_t dest;
    asm (
      "ld.global.L1::evict_first.b16 %0, [%1];"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_is_not_supported_before_SM_70__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_first.L2::256B.b16 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.L1::evict_first.L2::256B.b16 %0, [%1];"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_first.b32 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_evict_first(
  cuda_ptx::space_global_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_is_not_supported_before_SM_70__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_evict_first(
  cuda_ptx::space_global_t,
  const B32* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint32_t dest;
    asm (
      "ld.global.L1::evict_first.b32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_is_not_supported_before_SM_70__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_first.L2::256B.b32 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L1::evict_first.L2::256B.b32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_first.b64 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_evict_first(
  cuda_ptx::space_global_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_is_not_supported_before_SM_70__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_evict_first(
  cuda_ptx::space_global_t,
  const B64* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint64_t dest;
    asm (
      "ld.global.L1::evict_first.b64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_is_not_supported_before_SM_70__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_first.L2::256B.b64 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.L1::evict_first.L2::256B.b64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_first.b128 dest, [addr]; // PTX ISA 83, SM_70
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_evict_first(
  cuda_ptx::space_global_t,
  const B128* addr);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_is_not_supported_before_SM_70__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_evict_first(
  cuda_ptx::space_global_t,
  const B128* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.L1::evict_first.b128 B128_dest, [%2]; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_is_not_supported_before_SM_70__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.L1::evict_first.L2::256B.b128 dest, [addr]; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.L1::evict_first.L2::256B.b128 B128_dest, [%2]; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.L1::evict_first.v4.b64 dest, [addr]; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_L1_evict_first(
  cuda_ptx::space_global_t,
  const B256* addr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_L1_evict_first(
  cuda_ptx::space_global_t,
  const B256* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    cuda_ptx::longlong4_32a dest;
    asm (
      "ld.global.L1::evict_first.v4.b64 {%0, %1, %2, %3}, [%4];"
      : "=l"(dest.x),
        "=l"(dest.y),
        "=l"(dest.z),
        "=l"(dest.w)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B256*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_is_not_supported_before_SM_100__();
    cuda_ptx::longlong4_32a err_out_var{0, 0, 0, 0};
    return *reinterpret_cast<B256*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// ld.space.L1::evict_first.L2::cache_hint.b8 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L1::evict_first.L2::cache_hint.b8 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_first.L2::cache_hint.L2::256B.b8 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L1::evict_first.L2::cache_hint.L2::256B.b8 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_first.L2::cache_hint.b16 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.L1::evict_first.L2::cache_hint.b16 %0, [%1], %2;"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_first.L2::cache_hint.L2::256B.b16 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.L1::evict_first.L2::cache_hint.L2::256B.b16 %0, [%1], %2;"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_first.L2::cache_hint.b32 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L1::evict_first.L2::cache_hint.b32 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_first.L2::cache_hint.L2::256B.b32 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L1::evict_first.L2::cache_hint.L2::256B.b32 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_first.L2::cache_hint.b64 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.L1::evict_first.L2::cache_hint.b64 %0, [%1], %2;"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_first.L2::cache_hint.L2::256B.b64 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.L1::evict_first.L2::cache_hint.L2::256B.b64 %0, [%1], %2;"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_first.L2::cache_hint.b128 dest, [addr], cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.L1::evict_first.L2::cache_hint.b128 B128_dest, [%2], %3; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.L1::evict_first.L2::cache_hint.L2::256B.b128 dest, [addr], cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.L1::evict_first.L2::cache_hint.L2::256B.b128 B128_dest, [%2], %3; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.L1::evict_first.L2::cache_hint.v4.b64 dest, [addr], cache_policy; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B256* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_ld_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B256* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    cuda_ptx::longlong4_32a dest;
    asm (
      "ld.global.L1::evict_first.L2::cache_hint.v4.b64 {%0, %1, %2, %3}, [%4], %5;"
      : "=l"(dest.x),
        "=l"(dest.y),
        "=l"(dest.z),
        "=l"(dest.w)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B256*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_100__();
    cuda_ptx::longlong4_32a err_out_var{0, 0, 0, 0};
    return *reinterpret_cast<B256*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// ld.space.L1::evict_last.b8 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_evict_last(
  cuda_ptx::space_global_t,
  const B8* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_is_not_supported_before_SM_70__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_evict_last(
  cuda_ptx::space_global_t,
  const B8* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint32_t dest;
    asm (
      "ld.global.L1::evict_last.b8 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_is_not_supported_before_SM_70__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_last.L2::256B.b8 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L1::evict_last.L2::256B.b8 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_last.b16 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_evict_last(
  cuda_ptx::space_global_t,
  const B16* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_is_not_supported_before_SM_70__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_evict_last(
  cuda_ptx::space_global_t,
  const B16* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint16_t dest;
    asm (
      "ld.global.L1::evict_last.b16 %0, [%1];"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_is_not_supported_before_SM_70__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_last.L2::256B.b16 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.L1::evict_last.L2::256B.b16 %0, [%1];"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_last.b32 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_evict_last(
  cuda_ptx::space_global_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_is_not_supported_before_SM_70__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_evict_last(
  cuda_ptx::space_global_t,
  const B32* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint32_t dest;
    asm (
      "ld.global.L1::evict_last.b32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_is_not_supported_before_SM_70__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_last.L2::256B.b32 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L1::evict_last.L2::256B.b32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_last.b64 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_evict_last(
  cuda_ptx::space_global_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_is_not_supported_before_SM_70__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_evict_last(
  cuda_ptx::space_global_t,
  const B64* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint64_t dest;
    asm (
      "ld.global.L1::evict_last.b64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_is_not_supported_before_SM_70__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_last.L2::256B.b64 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.L1::evict_last.L2::256B.b64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_last.b128 dest, [addr]; // PTX ISA 83, SM_70
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_evict_last(
  cuda_ptx::space_global_t,
  const B128* addr);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_is_not_supported_before_SM_70__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_evict_last(
  cuda_ptx::space_global_t,
  const B128* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.L1::evict_last.b128 B128_dest, [%2]; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_is_not_supported_before_SM_70__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.L1::evict_last.L2::256B.b128 dest, [addr]; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.L1::evict_last.L2::256B.b128 B128_dest, [%2]; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.L1::evict_last.v4.b64 dest, [addr]; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_L1_evict_last(
  cuda_ptx::space_global_t,
  const B256* addr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_L1_evict_last(
  cuda_ptx::space_global_t,
  const B256* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    cuda_ptx::longlong4_32a dest;
    asm (
      "ld.global.L1::evict_last.v4.b64 {%0, %1, %2, %3}, [%4];"
      : "=l"(dest.x),
        "=l"(dest.y),
        "=l"(dest.z),
        "=l"(dest.w)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B256*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_is_not_supported_before_SM_100__();
    cuda_ptx::longlong4_32a err_out_var{0, 0, 0, 0};
    return *reinterpret_cast<B256*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// ld.space.L1::evict_last.L2::cache_hint.b8 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L1::evict_last.L2::cache_hint.b8 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_last.L2::cache_hint.L2::256B.b8 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L1::evict_last.L2::cache_hint.L2::256B.b8 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_last.L2::cache_hint.b16 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.L1::evict_last.L2::cache_hint.b16 %0, [%1], %2;"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_last.L2::cache_hint.L2::256B.b16 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.L1::evict_last.L2::cache_hint.L2::256B.b16 %0, [%1], %2;"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_last.L2::cache_hint.b32 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L1::evict_last.L2::cache_hint.b32 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_last.L2::cache_hint.L2::256B.b32 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L1::evict_last.L2::cache_hint.L2::256B.b32 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_last.L2::cache_hint.b64 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.L1::evict_last.L2::cache_hint.b64 %0, [%1], %2;"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_last.L2::cache_hint.L2::256B.b64 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.L1::evict_last.L2::cache_hint.L2::256B.b64 %0, [%1], %2;"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::evict_last.L2::cache_hint.b128 dest, [addr], cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.L1::evict_last.L2::cache_hint.b128 B128_dest, [%2], %3; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.L1::evict_last.L2::cache_hint.L2::256B.b128 dest, [addr], cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.L1::evict_last.L2::cache_hint.L2::256B.b128 B128_dest, [%2], %3; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.L1::evict_last.L2::cache_hint.v4.b64 dest, [addr], cache_policy; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B256* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_ld_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B256* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    cuda_ptx::longlong4_32a dest;
    asm (
      "ld.global.L1::evict_last.L2::cache_hint.v4.b64 {%0, %1, %2, %3}, [%4], %5;"
      : "=l"(dest.x),
        "=l"(dest.y),
        "=l"(dest.z),
        "=l"(dest.w)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B256*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_100__();
    cuda_ptx::longlong4_32a err_out_var{0, 0, 0, 0};
    return *reinterpret_cast<B256*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// ld.space.L1::no_allocate.b8 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B8* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_is_not_supported_before_SM_70__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B8* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint32_t dest;
    asm (
      "ld.global.L1::no_allocate.b8 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_is_not_supported_before_SM_70__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::no_allocate.L2::256B.b8 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L1::no_allocate.L2::256B.b8 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::no_allocate.b16 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B16* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_is_not_supported_before_SM_70__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B16* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint16_t dest;
    asm (
      "ld.global.L1::no_allocate.b16 %0, [%1];"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_is_not_supported_before_SM_70__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::no_allocate.L2::256B.b16 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.L1::no_allocate.L2::256B.b16 %0, [%1];"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::no_allocate.b32 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_is_not_supported_before_SM_70__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B32* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint32_t dest;
    asm (
      "ld.global.L1::no_allocate.b32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_is_not_supported_before_SM_70__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::no_allocate.L2::256B.b32 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L1::no_allocate.L2::256B.b32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::no_allocate.b64 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_is_not_supported_before_SM_70__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B64* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint64_t dest;
    asm (
      "ld.global.L1::no_allocate.b64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_is_not_supported_before_SM_70__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::no_allocate.L2::256B.b64 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.L1::no_allocate.L2::256B.b64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::no_allocate.b128 dest, [addr]; // PTX ISA 83, SM_70
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B128* addr);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_is_not_supported_before_SM_70__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B128* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.L1::no_allocate.b128 B128_dest, [%2]; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_is_not_supported_before_SM_70__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.L1::no_allocate.L2::256B.b128 dest, [addr]; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.L1::no_allocate.L2::256B.b128 B128_dest, [%2]; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.L1::no_allocate.v4.b64 dest, [addr]; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B256* addr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B256* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    cuda_ptx::longlong4_32a dest;
    asm (
      "ld.global.L1::no_allocate.v4.b64 {%0, %1, %2, %3}, [%4];"
      : "=l"(dest.x),
        "=l"(dest.y),
        "=l"(dest.z),
        "=l"(dest.w)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B256*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_is_not_supported_before_SM_100__();
    cuda_ptx::longlong4_32a err_out_var{0, 0, 0, 0};
    return *reinterpret_cast<B256*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// ld.space.L1::no_allocate.L2::cache_hint.b8 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L1::no_allocate.L2::cache_hint.b8 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::no_allocate.L2::cache_hint.L2::256B.b8 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L1::no_allocate.L2::cache_hint.L2::256B.b8 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::no_allocate.L2::cache_hint.b16 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.L1::no_allocate.L2::cache_hint.b16 %0, [%1], %2;"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::no_allocate.L2::cache_hint.L2::256B.b16 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.L1::no_allocate.L2::cache_hint.L2::256B.b16 %0, [%1], %2;"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::no_allocate.L2::cache_hint.b32 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L1::no_allocate.L2::cache_hint.b32 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::no_allocate.L2::cache_hint.L2::256B.b32 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.L1::no_allocate.L2::cache_hint.L2::256B.b32 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::no_allocate.L2::cache_hint.b64 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.L1::no_allocate.L2::cache_hint.b64 %0, [%1], %2;"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::no_allocate.L2::cache_hint.L2::256B.b64 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.L1::no_allocate.L2::cache_hint.L2::256B.b64 %0, [%1], %2;"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.L1::no_allocate.L2::cache_hint.b128 dest, [addr], cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.L1::no_allocate.L2::cache_hint.b128 B128_dest, [%2], %3; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.L1::no_allocate.L2::cache_hint.L2::256B.b128 dest, [addr], cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.L1::no_allocate.L2::cache_hint.L2::256B.b128 B128_dest, [%2], %3; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.L1::no_allocate.L2::cache_hint.v4.b64 dest, [addr], cache_policy; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B256* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B256* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    cuda_ptx::longlong4_32a dest;
    asm (
      "ld.global.L1::no_allocate.L2::cache_hint.v4.b64 {%0, %1, %2, %3}, [%4], %5;"
      : "=l"(dest.x),
        "=l"(dest.y),
        "=l"(dest.z),
        "=l"(dest.w)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B256*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_100__();
    cuda_ptx::longlong4_32a err_out_var{0, 0, 0, 0};
    return *reinterpret_cast<B256*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// ld.space.nc.b8 dest, [addr]; // PTX ISA 10, SM_50
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc(
  cuda_ptx::space_global_t,
  const B8* addr);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_ld_nc_is_not_supported_before_SM_50__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc(
  cuda_ptx::space_global_t,
  const B8* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "ld.global.nc.b8 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_is_not_supported_before_SM_50__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 100

/*
// ld.space.nc.L2::256B.b8 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L2_256B_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L2::256B.b8 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.b16 dest, [addr]; // PTX ISA 10, SM_50
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc(
  cuda_ptx::space_global_t,
  const B16* addr);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_ld_nc_is_not_supported_before_SM_50__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc(
  cuda_ptx::space_global_t,
  const B16* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint16_t dest;
    asm (
      "ld.global.nc.b16 %0, [%1];"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_is_not_supported_before_SM_50__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 100

/*
// ld.space.nc.L2::256B.b16 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L2_256B_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.nc.L2::256B.b16 %0, [%1];"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L2_256B_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.b32 dest, [addr]; // PTX ISA 10, SM_50
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc(
  cuda_ptx::space_global_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_ld_nc_is_not_supported_before_SM_50__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc(
  cuda_ptx::space_global_t,
  const B32* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "ld.global.nc.b32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_is_not_supported_before_SM_50__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 100

/*
// ld.space.nc.L2::256B.b32 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L2_256B_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L2::256B.b32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.b64 dest, [addr]; // PTX ISA 10, SM_50
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc(
  cuda_ptx::space_global_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_ld_nc_is_not_supported_before_SM_50__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc(
  cuda_ptx::space_global_t,
  const B64* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint64_t dest;
    asm (
      "ld.global.nc.b64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_is_not_supported_before_SM_50__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 100

/*
// ld.space.nc.L2::256B.b64 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L2_256B_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.nc.L2::256B.b64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L2_256B_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.b128 dest, [addr]; // PTX ISA 83, SM_70
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc(
  cuda_ptx::space_global_t,
  const B128* addr);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_nc_is_not_supported_before_SM_70__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc(
  cuda_ptx::space_global_t,
  const B128* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.nc.b128 B128_dest, [%2]; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_is_not_supported_before_SM_70__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.nc.L2::256B.b128 dest, [addr]; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_nc_L2_256B_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.nc.L2::256B.b128 B128_dest, [%2]; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L2_256B_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.nc.v4.b64 dest, [addr]; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_nc(
  cuda_ptx::space_global_t,
  const B256* addr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_ld_nc_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_nc(
  cuda_ptx::space_global_t,
  const B256* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    cuda_ptx::longlong4_32a dest;
    asm (
      "ld.global.nc.v4.b64 {%0, %1, %2, %3}, [%4];"
      : "=l"(dest.x),
        "=l"(dest.y),
        "=l"(dest.z),
        "=l"(dest.w)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B256*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_is_not_supported_before_SM_100__();
    cuda_ptx::longlong4_32a err_out_var{0, 0, 0, 0};
    return *reinterpret_cast<B256*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// ld.space.nc.L2::cache_hint.b8 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L2::cache_hint.b8 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L2::cache_hint.L2::256B.b8 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L2::cache_hint.L2::256B.b8 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L2::cache_hint.b16 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.nc.L2::cache_hint.b16 %0, [%1], %2;"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L2::cache_hint.L2::256B.b16 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.nc.L2::cache_hint.L2::256B.b16 %0, [%1], %2;"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L2::cache_hint.b32 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L2::cache_hint.b32 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L2::cache_hint.L2::256B.b32 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L2::cache_hint.L2::256B.b32 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L2::cache_hint.b64 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.nc.L2::cache_hint.b64 %0, [%1], %2;"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L2::cache_hint.L2::256B.b64 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.nc.L2::cache_hint.L2::256B.b64 %0, [%1], %2;"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L2::cache_hint.b128 dest, [addr], cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_nc_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.nc.L2::cache_hint.b128 B128_dest, [%2], %3; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L2_cache_hint_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.nc.L2::cache_hint.L2::256B.b128 dest, [addr], cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_nc_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.nc.L2::cache_hint.L2::256B.b128 B128_dest, [%2], %3; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.nc.L2::cache_hint.v4.b64 dest, [addr], cache_policy; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_nc_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B256* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_ld_nc_L2_cache_hint_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_nc_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B256* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    cuda_ptx::longlong4_32a dest;
    asm (
      "ld.global.nc.L2::cache_hint.v4.b64 {%0, %1, %2, %3}, [%4], %5;"
      : "=l"(dest.x),
        "=l"(dest.y),
        "=l"(dest.z),
        "=l"(dest.w)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B256*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L2_cache_hint_is_not_supported_before_SM_100__();
    cuda_ptx::longlong4_32a err_out_var{0, 0, 0, 0};
    return *reinterpret_cast<B256*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// ld.space.nc.L1::evict_first.b8 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_evict_first(
  cuda_ptx::space_global_t,
  const B8* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_is_not_supported_before_SM_70__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_evict_first(
  cuda_ptx::space_global_t,
  const B8* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::evict_first.b8 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_is_not_supported_before_SM_70__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_first.L2::256B.b8 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::evict_first.L2::256B.b8 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_first.b16 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_evict_first(
  cuda_ptx::space_global_t,
  const B16* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_is_not_supported_before_SM_70__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_evict_first(
  cuda_ptx::space_global_t,
  const B16* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint16_t dest;
    asm (
      "ld.global.nc.L1::evict_first.b16 %0, [%1];"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_is_not_supported_before_SM_70__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_first.L2::256B.b16 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.nc.L1::evict_first.L2::256B.b16 %0, [%1];"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_first.b32 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_evict_first(
  cuda_ptx::space_global_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_is_not_supported_before_SM_70__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_evict_first(
  cuda_ptx::space_global_t,
  const B32* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::evict_first.b32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_is_not_supported_before_SM_70__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_first.L2::256B.b32 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::evict_first.L2::256B.b32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_first.b64 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_evict_first(
  cuda_ptx::space_global_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_is_not_supported_before_SM_70__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_evict_first(
  cuda_ptx::space_global_t,
  const B64* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint64_t dest;
    asm (
      "ld.global.nc.L1::evict_first.b64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_is_not_supported_before_SM_70__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_first.L2::256B.b64 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.nc.L1::evict_first.L2::256B.b64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_first.b128 dest, [addr]; // PTX ISA 83, SM_70
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_evict_first(
  cuda_ptx::space_global_t,
  const B128* addr);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_is_not_supported_before_SM_70__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_evict_first(
  cuda_ptx::space_global_t,
  const B128* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.nc.L1::evict_first.b128 B128_dest, [%2]; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_is_not_supported_before_SM_70__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.nc.L1::evict_first.L2::256B.b128 dest, [addr]; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_evict_first_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.nc.L1::evict_first.L2::256B.b128 B128_dest, [%2]; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_L2_256B_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.nc.L1::evict_first.v4.b64 dest, [addr]; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_nc_L1_evict_first(
  cuda_ptx::space_global_t,
  const B256* addr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_nc_L1_evict_first(
  cuda_ptx::space_global_t,
  const B256* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    cuda_ptx::longlong4_32a dest;
    asm (
      "ld.global.nc.L1::evict_first.v4.b64 {%0, %1, %2, %3}, [%4];"
      : "=l"(dest.x),
        "=l"(dest.y),
        "=l"(dest.z),
        "=l"(dest.w)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B256*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_is_not_supported_before_SM_100__();
    cuda_ptx::longlong4_32a err_out_var{0, 0, 0, 0};
    return *reinterpret_cast<B256*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// ld.space.nc.L1::evict_first.L2::cache_hint.b8 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::evict_first.L2::cache_hint.b8 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_first.L2::cache_hint.L2::256B.b8 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::evict_first.L2::cache_hint.L2::256B.b8 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_first.L2::cache_hint.b16 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.nc.L1::evict_first.L2::cache_hint.b16 %0, [%1], %2;"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_first.L2::cache_hint.L2::256B.b16 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.nc.L1::evict_first.L2::cache_hint.L2::256B.b16 %0, [%1], %2;"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_first.L2::cache_hint.b32 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::evict_first.L2::cache_hint.b32 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_first.L2::cache_hint.L2::256B.b32 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::evict_first.L2::cache_hint.L2::256B.b32 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_first.L2::cache_hint.b64 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.nc.L1::evict_first.L2::cache_hint.b64 %0, [%1], %2;"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_first.L2::cache_hint.L2::256B.b64 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.nc.L1::evict_first.L2::cache_hint.L2::256B.b64 %0, [%1], %2;"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_first.L2::cache_hint.b128 dest, [addr], cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.nc.L1::evict_first.L2::cache_hint.b128 B128_dest, [%2], %3; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.nc.L1::evict_first.L2::cache_hint.L2::256B.b128 dest, [addr], cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_evict_first_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.nc.L1::evict_first.L2::cache_hint.L2::256B.b128 B128_dest, [%2], %3; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.nc.L1::evict_first.L2::cache_hint.v4.b64 dest, [addr], cache_policy; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_nc_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B256* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_nc_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B256* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    cuda_ptx::longlong4_32a dest;
    asm (
      "ld.global.nc.L1::evict_first.L2::cache_hint.v4.b64 {%0, %1, %2, %3}, [%4], %5;"
      : "=l"(dest.x),
        "=l"(dest.y),
        "=l"(dest.z),
        "=l"(dest.w)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B256*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_100__();
    cuda_ptx::longlong4_32a err_out_var{0, 0, 0, 0};
    return *reinterpret_cast<B256*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// ld.space.nc.L1::evict_last.b8 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_evict_last(
  cuda_ptx::space_global_t,
  const B8* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_is_not_supported_before_SM_70__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_evict_last(
  cuda_ptx::space_global_t,
  const B8* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::evict_last.b8 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_is_not_supported_before_SM_70__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_last.L2::256B.b8 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::evict_last.L2::256B.b8 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_last.b16 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_evict_last(
  cuda_ptx::space_global_t,
  const B16* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_is_not_supported_before_SM_70__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_evict_last(
  cuda_ptx::space_global_t,
  const B16* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint16_t dest;
    asm (
      "ld.global.nc.L1::evict_last.b16 %0, [%1];"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_is_not_supported_before_SM_70__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_last.L2::256B.b16 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.nc.L1::evict_last.L2::256B.b16 %0, [%1];"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_last.b32 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_evict_last(
  cuda_ptx::space_global_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_is_not_supported_before_SM_70__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_evict_last(
  cuda_ptx::space_global_t,
  const B32* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::evict_last.b32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_is_not_supported_before_SM_70__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_last.L2::256B.b32 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::evict_last.L2::256B.b32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_last.b64 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_evict_last(
  cuda_ptx::space_global_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_is_not_supported_before_SM_70__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_evict_last(
  cuda_ptx::space_global_t,
  const B64* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint64_t dest;
    asm (
      "ld.global.nc.L1::evict_last.b64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_is_not_supported_before_SM_70__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_last.L2::256B.b64 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.nc.L1::evict_last.L2::256B.b64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_last.b128 dest, [addr]; // PTX ISA 83, SM_70
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_evict_last(
  cuda_ptx::space_global_t,
  const B128* addr);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_is_not_supported_before_SM_70__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_evict_last(
  cuda_ptx::space_global_t,
  const B128* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.nc.L1::evict_last.b128 B128_dest, [%2]; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_is_not_supported_before_SM_70__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.nc.L1::evict_last.L2::256B.b128 dest, [addr]; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_evict_last_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.nc.L1::evict_last.L2::256B.b128 B128_dest, [%2]; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_L2_256B_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.nc.L1::evict_last.v4.b64 dest, [addr]; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_nc_L1_evict_last(
  cuda_ptx::space_global_t,
  const B256* addr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_nc_L1_evict_last(
  cuda_ptx::space_global_t,
  const B256* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    cuda_ptx::longlong4_32a dest;
    asm (
      "ld.global.nc.L1::evict_last.v4.b64 {%0, %1, %2, %3}, [%4];"
      : "=l"(dest.x),
        "=l"(dest.y),
        "=l"(dest.z),
        "=l"(dest.w)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B256*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_is_not_supported_before_SM_100__();
    cuda_ptx::longlong4_32a err_out_var{0, 0, 0, 0};
    return *reinterpret_cast<B256*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// ld.space.nc.L1::evict_last.L2::cache_hint.b8 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::evict_last.L2::cache_hint.b8 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_last.L2::cache_hint.L2::256B.b8 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::evict_last.L2::cache_hint.L2::256B.b8 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_last.L2::cache_hint.b16 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.nc.L1::evict_last.L2::cache_hint.b16 %0, [%1], %2;"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_last.L2::cache_hint.L2::256B.b16 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.nc.L1::evict_last.L2::cache_hint.L2::256B.b16 %0, [%1], %2;"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_last.L2::cache_hint.b32 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::evict_last.L2::cache_hint.b32 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_last.L2::cache_hint.L2::256B.b32 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::evict_last.L2::cache_hint.L2::256B.b32 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_last.L2::cache_hint.b64 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.nc.L1::evict_last.L2::cache_hint.b64 %0, [%1], %2;"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_last.L2::cache_hint.L2::256B.b64 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.nc.L1::evict_last.L2::cache_hint.L2::256B.b64 %0, [%1], %2;"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::evict_last.L2::cache_hint.b128 dest, [addr], cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.nc.L1::evict_last.L2::cache_hint.b128 B128_dest, [%2], %3; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.nc.L1::evict_last.L2::cache_hint.L2::256B.b128 dest, [addr], cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_evict_last_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.nc.L1::evict_last.L2::cache_hint.L2::256B.b128 B128_dest, [%2], %3; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.nc.L1::evict_last.L2::cache_hint.v4.b64 dest, [addr], cache_policy; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_nc_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B256* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_nc_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B256* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    cuda_ptx::longlong4_32a dest;
    asm (
      "ld.global.nc.L1::evict_last.L2::cache_hint.v4.b64 {%0, %1, %2, %3}, [%4], %5;"
      : "=l"(dest.x),
        "=l"(dest.y),
        "=l"(dest.z),
        "=l"(dest.w)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B256*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_100__();
    cuda_ptx::longlong4_32a err_out_var{0, 0, 0, 0};
    return *reinterpret_cast<B256*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// ld.space.nc.L1::no_allocate.b8 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B8* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_is_not_supported_before_SM_70__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B8* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::no_allocate.b8 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_is_not_supported_before_SM_70__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::no_allocate.L2::256B.b8 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::no_allocate.L2::256B.b8 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::no_allocate.b16 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B16* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_is_not_supported_before_SM_70__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B16* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint16_t dest;
    asm (
      "ld.global.nc.L1::no_allocate.b16 %0, [%1];"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_is_not_supported_before_SM_70__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::no_allocate.L2::256B.b16 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.nc.L1::no_allocate.L2::256B.b16 %0, [%1];"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::no_allocate.b32 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_is_not_supported_before_SM_70__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B32* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::no_allocate.b32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_is_not_supported_before_SM_70__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::no_allocate.L2::256B.b32 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::no_allocate.L2::256B.b32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::no_allocate.b64 dest, [addr]; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_is_not_supported_before_SM_70__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B64* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    std::uint64_t dest;
    asm (
      "ld.global.nc.L1::no_allocate.b64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_is_not_supported_before_SM_70__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::no_allocate.L2::256B.b64 dest, [addr]; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.nc.L1::no_allocate.L2::256B.b64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::no_allocate.b128 dest, [addr]; // PTX ISA 83, SM_70
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B128* addr);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_is_not_supported_before_SM_70__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B128* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.nc.L1::no_allocate.b128 B128_dest, [%2]; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_is_not_supported_before_SM_70__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.nc.L1::no_allocate.L2::256B.b128 dest, [addr]; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_no_allocate_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.nc.L1::no_allocate.L2::256B.b128 B128_dest, [%2]; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_L2_256B_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.nc.L1::no_allocate.v4.b64 dest, [addr]; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_nc_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B256* addr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_nc_L1_no_allocate(
  cuda_ptx::space_global_t,
  const B256* addr)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    cuda_ptx::longlong4_32a dest;
    asm (
      "ld.global.nc.L1::no_allocate.v4.b64 {%0, %1, %2, %3}, [%4];"
      : "=l"(dest.x),
        "=l"(dest.y),
        "=l"(dest.z),
        "=l"(dest.w)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B256*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_is_not_supported_before_SM_100__();
    cuda_ptx::longlong4_32a err_out_var{0, 0, 0, 0};
    return *reinterpret_cast<B256*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// ld.space.nc.L1::no_allocate.L2::cache_hint.b8 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::no_allocate.L2::cache_hint.b8 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::no_allocate.L2::cache_hint.L2::256B.b8 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline B8 ld_nc_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B8* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::no_allocate.L2::cache_hint.L2::256B.b8 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return __u32_as_b8<B8>(dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B8*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::no_allocate.L2::cache_hint.b16 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.nc.L1::no_allocate.L2::cache_hint.b16 %0, [%1], %2;"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::no_allocate.L2::cache_hint.L2::256B.b16 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 ld_nc_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B16* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint16_t dest;
    asm (
      "ld.global.nc.L1::no_allocate.L2::cache_hint.L2::256B.b16 %0, [%1], %2;"
      : "=h"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::no_allocate.L2::cache_hint.b32 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::no_allocate.L2::cache_hint.b32 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::no_allocate.L2::cache_hint.L2::256B.b32 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 ld_nc_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B32* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t dest;
    asm (
      "ld.global.nc.L1::no_allocate.L2::cache_hint.L2::256B.b32 %0, [%1], %2;"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::no_allocate.L2::cache_hint.b64 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.nc.L1::no_allocate.L2::cache_hint.b64 %0, [%1], %2;"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::no_allocate.L2::cache_hint.L2::256B.b64 dest, [addr], cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 ld_nc_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B64* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t dest;
    asm (
      "ld.global.nc.L1::no_allocate.L2::cache_hint.L2::256B.b64 %0, [%1], %2;"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// ld.space.nc.L1::no_allocate.L2::cache_hint.b128 dest, [addr], cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.nc.L1::no_allocate.L2::cache_hint.b128 B128_dest, [%2], %3; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.nc.L1::no_allocate.L2::cache_hint.L2::256B.b128 dest, [addr], cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline B128 ld_nc_L1_no_allocate_L2_cache_hint_L2_256B(
  cuda_ptx::space_global_t,
  const B128* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    longlong2 dest;
    asm (
      "{\n\t"
      ".reg .b128 B128_dest; \n\t"
      "ld.global.nc.L1::no_allocate.L2::cache_hint.L2::256B.b128 B128_dest, [%2], %3; \n\t"
      "mov.b128 {%0, %1}, B128_dest; \n"
      "}"
      : "=l"(dest.x),
        "=l"(dest.y)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B128*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_L2_256B_is_not_supported_before_SM_80__();
    longlong2 err_out_var{0, 0};
    return *reinterpret_cast<B128*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// ld.space.nc.L1::no_allocate.L2::cache_hint.v4.b64 dest, [addr], cache_policy; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_nc_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B256* addr,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline B256 ld_nc_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  const B256* addr,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    cuda_ptx::longlong4_32a dest;
    asm (
      "ld.global.nc.L1::no_allocate.L2::cache_hint.v4.b64 {%0, %1, %2, %3}, [%4], %5;"
      : "=l"(dest.x),
        "=l"(dest.y),
        "=l"(dest.z),
        "=l"(dest.w)
      : "l"(__as_ptr_gmem(addr)),
        "l"(cache_policy)
      : "memory"
    );
    return *reinterpret_cast<B256*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ld_nc_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_100__();
    cuda_ptx::longlong4_32a err_out_var{0, 0, 0, 0};
    return *reinterpret_cast<B256*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 880

} // namespace cuda_ptx

#endif // INSTRUCTIONS_LD_H

// End of ./instructions/ld.h
// Start of ./instructions/ldmatrix.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_LDMATRIX_H
#define INSTRUCTIONS_LDMATRIX_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/ldmatrix.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// ldmatrix.sync.aligned.m8n8.x1.space.b16 out, [smem_ptr]; // PTX ISA 65, SM_75
// .space     = { .shared }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void ldmatrix_m8n8(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[1],
  const B16* smem_ptr);
*/
#if __libcuda_ptx_isa >= 650
extern "C" __device__ void __cuda_ptx_ldmatrix_m8n8_is_not_supported_before_SM_75__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void ldmatrix_m8n8(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[1],
  const B16* smem_ptr)
{
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 750
    asm (
      "ldmatrix.sync.aligned.m8n8.x1.shared.b16 {%0}, [%1];"
      : "=r"(out[0])
      : "r"(__as_ptr_smem(smem_ptr))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ldmatrix_m8n8_is_not_supported_before_SM_75__();
  #endif
}
#endif // __libcuda_ptx_isa >= 650

/*
// ldmatrix.sync.aligned.m8n8.x2.space.b16 out, [smem_ptr]; // PTX ISA 65, SM_75
// .space     = { .shared }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void ldmatrix_m8n8(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[2],
  const B16* smem_ptr);
*/
#if __libcuda_ptx_isa >= 650
extern "C" __device__ void __cuda_ptx_ldmatrix_m8n8_is_not_supported_before_SM_75__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void ldmatrix_m8n8(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[2],
  const B16* smem_ptr)
{
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 750
    asm (
      "ldmatrix.sync.aligned.m8n8.x2.shared.b16 {%0, %1}, [%2];"
      : "=r"(out[0]),
        "=r"(out[1])
      : "r"(__as_ptr_smem(smem_ptr))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ldmatrix_m8n8_is_not_supported_before_SM_75__();
  #endif
}
#endif // __libcuda_ptx_isa >= 650

/*
// ldmatrix.sync.aligned.m8n8.x4.space.b16 out, [smem_ptr]; // PTX ISA 65, SM_75
// .space     = { .shared }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void ldmatrix_m8n8(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[4],
  const B16* smem_ptr);
*/
#if __libcuda_ptx_isa >= 650
extern "C" __device__ void __cuda_ptx_ldmatrix_m8n8_is_not_supported_before_SM_75__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void ldmatrix_m8n8(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[4],
  const B16* smem_ptr)
{
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 750
    asm (
      "ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0, %1, %2, %3}, [%4];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3])
      : "r"(__as_ptr_smem(smem_ptr))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ldmatrix_m8n8_is_not_supported_before_SM_75__();
  #endif
}
#endif // __libcuda_ptx_isa >= 650

/*
// ldmatrix.sync.aligned.m8n8.x1.trans.space.b16 out, [smem_ptr]; // PTX ISA 65, SM_75
// .space     = { .shared }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void ldmatrix_m8n8_trans(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[1],
  const B16* smem_ptr);
*/
#if __libcuda_ptx_isa >= 650
extern "C" __device__ void __cuda_ptx_ldmatrix_m8n8_trans_is_not_supported_before_SM_75__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void ldmatrix_m8n8_trans(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[1],
  const B16* smem_ptr)
{
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 750
    asm (
      "ldmatrix.sync.aligned.m8n8.x1.trans.shared.b16 {%0}, [%1];"
      : "=r"(out[0])
      : "r"(__as_ptr_smem(smem_ptr))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ldmatrix_m8n8_trans_is_not_supported_before_SM_75__();
  #endif
}
#endif // __libcuda_ptx_isa >= 650

/*
// ldmatrix.sync.aligned.m8n8.x2.trans.space.b16 out, [smem_ptr]; // PTX ISA 65, SM_75
// .space     = { .shared }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void ldmatrix_m8n8_trans(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[2],
  const B16* smem_ptr);
*/
#if __libcuda_ptx_isa >= 650
extern "C" __device__ void __cuda_ptx_ldmatrix_m8n8_trans_is_not_supported_before_SM_75__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void ldmatrix_m8n8_trans(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[2],
  const B16* smem_ptr)
{
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 750
    asm (
      "ldmatrix.sync.aligned.m8n8.x2.trans.shared.b16 {%0, %1}, [%2];"
      : "=r"(out[0]),
        "=r"(out[1])
      : "r"(__as_ptr_smem(smem_ptr))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ldmatrix_m8n8_trans_is_not_supported_before_SM_75__();
  #endif
}
#endif // __libcuda_ptx_isa >= 650

/*
// ldmatrix.sync.aligned.m8n8.x4.trans.space.b16 out, [smem_ptr]; // PTX ISA 65, SM_75
// .space     = { .shared }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void ldmatrix_m8n8_trans(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[4],
  const B16* smem_ptr);
*/
#if __libcuda_ptx_isa >= 650
extern "C" __device__ void __cuda_ptx_ldmatrix_m8n8_trans_is_not_supported_before_SM_75__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void ldmatrix_m8n8_trans(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[4],
  const B16* smem_ptr)
{
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 750
    asm (
      "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3])
      : "r"(__as_ptr_smem(smem_ptr))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ldmatrix_m8n8_trans_is_not_supported_before_SM_75__();
  #endif
}
#endif // __libcuda_ptx_isa >= 650

/*
// ldmatrix.sync.aligned.m8n16.x1.space.b8x16.b6x16_p32 out, [smem_ptr]; // PTX ISA 86, SM_100a, SM_110a
// .space     = { .shared }
template <typename = void>
__device__ static inline void ldmatrix_m8n16_b8x16_b6x16_p32(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[1],
  const void* smem_ptr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_ldmatrix_m8n16_b8x16_b6x16_p32_is_only_supported_on_SM_100a_110a__();
template <typename = void>
__device__ static inline void ldmatrix_m8n16_b8x16_b6x16_p32(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[1],
  const void* smem_ptr)
{
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    asm (
      "ldmatrix.sync.aligned.m8n16.x1.shared.b8x16.b6x16_p32 {%0}, [%1];"
      : "=r"(out[0])
      : "r"(__as_ptr_smem(smem_ptr))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ldmatrix_m8n16_b8x16_b6x16_p32_is_only_supported_on_SM_100a_110a__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// ldmatrix.sync.aligned.m8n16.x1.space.b8x16.b4x16_p64 out, [smem_ptr]; // PTX ISA 86, SM_100a, SM_110a
// .space     = { .shared }
template <typename = void>
__device__ static inline void ldmatrix_m8n16_b8x16_b4x16_p64(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[1],
  const void* smem_ptr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_ldmatrix_m8n16_b8x16_b4x16_p64_is_only_supported_on_SM_100a_110a__();
template <typename = void>
__device__ static inline void ldmatrix_m8n16_b8x16_b4x16_p64(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[1],
  const void* smem_ptr)
{
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    asm (
      "ldmatrix.sync.aligned.m8n16.x1.shared.b8x16.b4x16_p64 {%0}, [%1];"
      : "=r"(out[0])
      : "r"(__as_ptr_smem(smem_ptr))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ldmatrix_m8n16_b8x16_b4x16_p64_is_only_supported_on_SM_100a_110a__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// ldmatrix.sync.aligned.m8n16.x2.space.b8x16.b6x16_p32 out, [smem_ptr]; // PTX ISA 86, SM_100a, SM_110a
// .space     = { .shared }
template <typename = void>
__device__ static inline void ldmatrix_m8n16_b8x16_b6x16_p32(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[2],
  const void* smem_ptr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_ldmatrix_m8n16_b8x16_b6x16_p32_is_only_supported_on_SM_100a_110a__();
template <typename = void>
__device__ static inline void ldmatrix_m8n16_b8x16_b6x16_p32(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[2],
  const void* smem_ptr)
{
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    asm (
      "ldmatrix.sync.aligned.m8n16.x2.shared.b8x16.b6x16_p32 {%0, %1}, [%2];"
      : "=r"(out[0]),
        "=r"(out[1])
      : "r"(__as_ptr_smem(smem_ptr))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ldmatrix_m8n16_b8x16_b6x16_p32_is_only_supported_on_SM_100a_110a__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// ldmatrix.sync.aligned.m8n16.x2.space.b8x16.b4x16_p64 out, [smem_ptr]; // PTX ISA 86, SM_100a, SM_110a
// .space     = { .shared }
template <typename = void>
__device__ static inline void ldmatrix_m8n16_b8x16_b4x16_p64(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[2],
  const void* smem_ptr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_ldmatrix_m8n16_b8x16_b4x16_p64_is_only_supported_on_SM_100a_110a__();
template <typename = void>
__device__ static inline void ldmatrix_m8n16_b8x16_b4x16_p64(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[2],
  const void* smem_ptr)
{
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    asm (
      "ldmatrix.sync.aligned.m8n16.x2.shared.b8x16.b4x16_p64 {%0, %1}, [%2];"
      : "=r"(out[0]),
        "=r"(out[1])
      : "r"(__as_ptr_smem(smem_ptr))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ldmatrix_m8n16_b8x16_b4x16_p64_is_only_supported_on_SM_100a_110a__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// ldmatrix.sync.aligned.m8n16.x4.space.b8x16.b6x16_p32 out, [smem_ptr]; // PTX ISA 86, SM_100a, SM_110a
// .space     = { .shared }
template <typename = void>
__device__ static inline void ldmatrix_m8n16_b8x16_b6x16_p32(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[4],
  const void* smem_ptr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_ldmatrix_m8n16_b8x16_b6x16_p32_is_only_supported_on_SM_100a_110a__();
template <typename = void>
__device__ static inline void ldmatrix_m8n16_b8x16_b6x16_p32(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[4],
  const void* smem_ptr)
{
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    asm (
      "ldmatrix.sync.aligned.m8n16.x4.shared.b8x16.b6x16_p32 {%0, %1, %2, %3}, [%4];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3])
      : "r"(__as_ptr_smem(smem_ptr))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ldmatrix_m8n16_b8x16_b6x16_p32_is_only_supported_on_SM_100a_110a__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// ldmatrix.sync.aligned.m8n16.x4.space.b8x16.b4x16_p64 out, [smem_ptr]; // PTX ISA 86, SM_100a, SM_110a
// .space     = { .shared }
template <typename = void>
__device__ static inline void ldmatrix_m8n16_b8x16_b4x16_p64(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[4],
  const void* smem_ptr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_ldmatrix_m8n16_b8x16_b4x16_p64_is_only_supported_on_SM_100a_110a__();
template <typename = void>
__device__ static inline void ldmatrix_m8n16_b8x16_b4x16_p64(
  cuda_ptx::space_shared_t,
  std::uint32_t (&out)[4],
  const void* smem_ptr)
{
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    asm (
      "ldmatrix.sync.aligned.m8n16.x4.shared.b8x16.b4x16_p64 {%0, %1, %2, %3}, [%4];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3])
      : "r"(__as_ptr_smem(smem_ptr))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_ldmatrix_m8n16_b8x16_b4x16_p64_is_only_supported_on_SM_100a_110a__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

} // namespace cuda_ptx

#endif // INSTRUCTIONS_LDMATRIX_H

// End of ./instructions/ldmatrix.h
// Start of ./instructions/mapa.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_MAPA_H
#define INSTRUCTIONS_MAPA_H

#include <nv/target>            // NV_IF_TARGET below.

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/mapa.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// mapa{.space}.u32  dest, addr, target_cta; // PTX ISA 78, SM_90
// .space     = { .shared::cluster }
template <typename Tp>
__device__ static inline Tp* mapa(
  cuda_ptx::space_cluster_t,
  const Tp* addr,
  std::uint32_t target_cta);
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_mapa_is_not_supported_before_SM_90__();
template <typename Tp>
__device__ static inline Tp* mapa(
  cuda_ptx::space_cluster_t,
  const Tp* addr,
  std::uint32_t target_cta)
{
  // space == space_cluster (due to parameter type constraint)
  NV_IF_ELSE_TARGET(NV_PROVIDES_SM_90, (
    return static_cast<Tp*>(__cluster_map_shared_rank(addr, target_cta));
  ),(
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mapa_is_not_supported_before_SM_90__();
    return __from_ptr_dsmem<Tp>(0);
  ));
}
#endif // __libcuda_ptx_isa >= 780

} // namespace cuda_ptx

#endif // INSTRUCTIONS_MAPA_H

// End of ./instructions/mapa.h
// Start of ./instructions/mbarrier_arrive.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_MBARRIER_ARRIVE_H
#define INSTRUCTIONS_MBARRIER_ARRIVE_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/mbarrier_arrive.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// mbarrier.arrive.shared.b64                                  state,  [addr];           // 1.  PTX ISA 70, SM_80
template <typename = void>
__device__ static inline std::uint64_t mbarrier_arrive(
  std::uint64_t* addr);
*/
#if __libcuda_ptx_isa >= 700
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_80__();
template <typename = void>
__device__ static inline std::uint64_t mbarrier_arrive(
  std::uint64_t* addr)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t state;
    asm (
      "mbarrier.arrive.shared.b64                                  %0,  [%1];           // 1. "
      : "=l"(state)
      : "r"(__as_ptr_smem(addr))
      : "memory"
    );
    return state;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_80__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 700

/*
// mbarrier.arrive.shared::cta.b64                             state,  [addr], count;    // 2.  PTX ISA 78, SM_90
template <typename = void>
__device__ static inline std::uint64_t mbarrier_arrive(
  std::uint64_t* addr,
  const std::uint32_t& count);
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint64_t mbarrier_arrive(
  std::uint64_t* addr,
  const std::uint32_t& count)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t state;
    asm (
      "mbarrier.arrive.shared::cta.b64                             %0,  [%1], %2;    // 2. "
      : "=l"(state)
      : "r"(__as_ptr_smem(addr)),
        "r"(count)
      : "memory"
    );
    return state;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mbarrier.arrive.sem.scope.space.b64                   state,  [addr];           // 3a.  PTX ISA 80, SM_90
// .sem       = { .release }
// .scope     = { .cta, .cluster }
// .space     = { .shared::cta }
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr)
{
  // sem == sem_release (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t state;
    if constexpr (scope == scope_cta) {
      asm (
        "mbarrier.arrive.release.cta.shared::cta.b64                   %0,  [%1];           // 3a. "
        : "=l"(state)
        : "r"(__as_ptr_smem(addr))
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "mbarrier.arrive.release.cluster.shared::cta.b64                   %0,  [%1];           // 3a. "
        : "=l"(state)
        : "r"(__as_ptr_smem(addr))
        : "memory"
      );
    }
    return state;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// mbarrier.arrive.sem.scope.space.b64                   state,  [addr], count;    // 3b.  PTX ISA 80, SM_90
// .sem       = { .release }
// .scope     = { .cta, .cluster }
// .space     = { .shared::cta }
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  const std::uint32_t& count);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  const std::uint32_t& count)
{
  // sem == sem_release (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t state;
    if constexpr (scope == scope_cta) {
      asm (
        "mbarrier.arrive.release.cta.shared::cta.b64                   %0,  [%1], %2;    // 3b. "
        : "=l"(state)
        : "r"(__as_ptr_smem(addr)),
          "r"(count)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "mbarrier.arrive.release.cluster.shared::cta.b64                   %0,  [%1], %2;    // 3b. "
        : "=l"(state)
        : "r"(__as_ptr_smem(addr)),
          "r"(count)
        : "memory"
      );
    }
    return state;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// mbarrier.arrive.sem.scope.space.b64                   _, [addr];                // 4a.  PTX ISA 80, SM_90
// .sem       = { .release }
// .scope     = { .cluster }
// .space     = { .shared::cluster }
template <typename = void>
__device__ static inline void mbarrier_arrive(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void mbarrier_arrive(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr)
{
  // sem == sem_release (due to parameter type constraint)
  // scope == scope_cluster (due to parameter type constraint)
  // space == space_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "mbarrier.arrive.release.cluster.shared::cluster.b64                   _, [%0];                // 4a. "
      :
      : "r"(__as_ptr_remote_dsmem(addr))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// mbarrier.arrive.sem.scope.space.b64                   _, [addr], count;         // 4b.  PTX ISA 80, SM_90
// .sem       = { .release }
// .scope     = { .cluster }
// .space     = { .shared::cluster }
template <typename = void>
__device__ static inline void mbarrier_arrive(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  const std::uint32_t& count);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void mbarrier_arrive(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  const std::uint32_t& count)
{
  // sem == sem_release (due to parameter type constraint)
  // scope == scope_cluster (due to parameter type constraint)
  // space == space_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "mbarrier.arrive.release.cluster.shared::cluster.b64                   _, [%0], %1;         // 4b. "
      :
      : "r"(__as_ptr_remote_dsmem(addr)),
        "r"(count)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// mbarrier.arrive.sem.scope.space.b64 state, [addr], count; // PTX ISA 86, SM_90
// .sem       = { .relaxed }
// .scope     = { .cta, .cluster }
// .space     = { .shared::cta }
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  const std::uint32_t& count);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  const std::uint32_t& count)
{
  // sem == sem_relaxed (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t state;
    if constexpr (scope == scope_cta) {
      asm (
        "mbarrier.arrive.relaxed.cta.shared::cta.b64 %0, [%1], %2;"
        : "=l"(state)
        : "r"(__as_ptr_smem(addr)),
          "r"(count)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "mbarrier.arrive.relaxed.cluster.shared::cta.b64 %0, [%1], %2;"
        : "=l"(state)
        : "r"(__as_ptr_smem(addr)),
          "r"(count)
        : "memory"
      );
    }
    return state;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// mbarrier.arrive.sem.scope.space.b64 state, [addr]; // PTX ISA 86, SM_90
// .sem       = { .relaxed }
// .scope     = { .cta, .cluster }
// .space     = { .shared::cta }
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr)
{
  // sem == sem_relaxed (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t state;
    if constexpr (scope == scope_cta) {
      asm (
        "mbarrier.arrive.relaxed.cta.shared::cta.b64 %0, [%1];"
        : "=l"(state)
        : "r"(__as_ptr_smem(addr))
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "mbarrier.arrive.relaxed.cluster.shared::cta.b64 %0, [%1];"
        : "=l"(state)
        : "r"(__as_ptr_smem(addr))
        : "memory"
      );
    }
    return state;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// mbarrier.arrive.sem.scope.space.b64 _, [addr], count; // PTX ISA 86, SM_90
// .sem       = { .relaxed }
// .scope     = { .cluster }
// .space     = { .shared::cluster }
template <typename = void>
__device__ static inline void mbarrier_arrive(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  const std::uint32_t& count);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void mbarrier_arrive(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  const std::uint32_t& count)
{
  // sem == sem_relaxed (due to parameter type constraint)
  // scope == scope_cluster (due to parameter type constraint)
  // space == space_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "mbarrier.arrive.relaxed.cluster.shared::cluster.b64 _, [%0], %1;"
      :
      : "r"(__as_ptr_smem(addr)),
        "r"(count)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// mbarrier.arrive.sem.scope.space.b64 _, [addr]; // PTX ISA 86, SM_90
// .sem       = { .relaxed }
// .scope     = { .cluster }
// .space     = { .shared::cluster }
template <typename = void>
__device__ static inline void mbarrier_arrive(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void mbarrier_arrive(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr)
{
  // sem == sem_relaxed (due to parameter type constraint)
  // scope == scope_cluster (due to parameter type constraint)
  // space == space_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "mbarrier.arrive.relaxed.cluster.shared::cluster.b64 _, [%0];"
      :
      : "r"(__as_ptr_smem(addr))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860
/*
// mbarrier.arrive.noComplete.shared.b64                       state,  [addr], count;    // 5.  PTX ISA 70, SM_80
template <typename = void>
__device__ static inline std::uint64_t mbarrier_arrive_no_complete(
  std::uint64_t* addr,
  const std::uint32_t& count);
*/
#if __libcuda_ptx_isa >= 700
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_no_complete_is_not_supported_before_SM_80__();
template <typename = void>
__device__ static inline std::uint64_t mbarrier_arrive_no_complete(
  std::uint64_t* addr,
  const std::uint32_t& count)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t state;
    asm (
      "mbarrier.arrive.noComplete.shared.b64                       %0,  [%1], %2;    // 5. "
      : "=l"(state)
      : "r"(__as_ptr_smem(addr)),
        "r"(count)
      : "memory"
    );
    return state;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_no_complete_is_not_supported_before_SM_80__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 700
/*
// mbarrier.arrive.expect_tx.sem.scope.space.b64 state, [addr], tx_count; // 8.  PTX ISA 80, SM_90
// .sem       = { .release }
// .scope     = { .cta, .cluster }
// .space     = { .shared::cta }
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive_expect_tx(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  const std::uint32_t& tx_count);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive_expect_tx(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  const std::uint32_t& tx_count)
{
  // sem == sem_release (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t state;
    if constexpr (scope == scope_cta) {
      asm (
        "mbarrier.arrive.expect_tx.release.cta.shared::cta.b64 %0, [%1], %2; // 8. "
        : "=l"(state)
        : "r"(__as_ptr_smem(addr)),
          "r"(tx_count)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "mbarrier.arrive.expect_tx.release.cluster.shared::cta.b64 %0, [%1], %2; // 8. "
        : "=l"(state)
        : "r"(__as_ptr_smem(addr)),
          "r"(tx_count)
        : "memory"
      );
    }
    return state;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// mbarrier.arrive.expect_tx.sem.scope.space.b64   _, [addr], tx_count; // 9.  PTX ISA 80, SM_90
// .sem       = { .release }
// .scope     = { .cluster }
// .space     = { .shared::cluster }
template <typename = void>
__device__ static inline void mbarrier_arrive_expect_tx(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  const std::uint32_t& tx_count);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void mbarrier_arrive_expect_tx(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  const std::uint32_t& tx_count)
{
  // sem == sem_release (due to parameter type constraint)
  // scope == scope_cluster (due to parameter type constraint)
  // space == space_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "mbarrier.arrive.expect_tx.release.cluster.shared::cluster.b64   _, [%0], %1; // 9. "
      :
      : "r"(__as_ptr_remote_dsmem(addr)),
        "r"(tx_count)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// mbarrier.arrive.expect_tx.sem.scope.space.b64 state, [addr], txCount; // PTX ISA 86, SM_90
// .sem       = { .relaxed }
// .scope     = { .cta, .cluster }
// .space     = { .shared::cta }
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive_expect_tx(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  const std::uint32_t& txCount);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive_expect_tx(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  const std::uint32_t& txCount)
{
  // sem == sem_relaxed (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t state;
    if constexpr (scope == scope_cta) {
      asm (
        "mbarrier.arrive.expect_tx.relaxed.cta.shared::cta.b64 %0, [%1], %2;"
        : "=l"(state)
        : "r"(__as_ptr_dsmem(addr)),
          "r"(txCount)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "mbarrier.arrive.expect_tx.relaxed.cluster.shared::cta.b64 %0, [%1], %2;"
        : "=l"(state)
        : "r"(__as_ptr_dsmem(addr)),
          "r"(txCount)
        : "memory"
      );
    }
    return state;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// mbarrier.arrive.expect_tx.sem.scope.space.b64 _, [addr], txCount; // PTX ISA 86, SM_90
// .sem       = { .relaxed }
// .scope     = { .cluster }
// .space     = { .shared::cluster }
template <typename = void>
__device__ static inline void mbarrier_arrive_expect_tx(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  const std::uint32_t& txCount);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void mbarrier_arrive_expect_tx(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  const std::uint32_t& txCount)
{
  // sem == sem_relaxed (due to parameter type constraint)
  // scope == scope_cluster (due to parameter type constraint)
  // space == space_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "mbarrier.arrive.expect_tx.relaxed.cluster.shared::cluster.b64 _, [%0], %1;"
      :
      : "r"(__as_ptr_dsmem(addr)),
        "r"(txCount)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_expect_tx_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

} // namespace cuda_ptx

#endif // INSTRUCTIONS_MBARRIER_ARRIVE_H

// End of ./instructions/mbarrier_arrive.h
// Start of ./instructions/mbarrier_arrive_drop.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_MBARRIER_ARRIVE_DROP_H
#define INSTRUCTIONS_MBARRIER_ARRIVE_DROP_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/mbarrier_arrive_drop.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// mbarrier.arrive_drop.sem.scope.space.b64 state, [addr], count; // PTX ISA 80, SM_90
// .sem       = { .release }
// .scope     = { .cta, .cluster }
// .space     = { .shared::cta }
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive_drop(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  std::uint32_t count);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_drop_is_only_supported_on_SM_90_depending_on_the_variant__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive_drop(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  std::uint32_t count)
{
  // sem == sem_release (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t state;
    if constexpr (scope == scope_cta) {
      asm (
        "mbarrier.arrive_drop.release.cta.shared::cta.b64 %0, [%1], %2;"
        : "=l"(state)
        : "r"(__as_ptr_smem(addr)),
          "r"(count)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "mbarrier.arrive_drop.release.cluster.shared::cta.b64 %0, [%1], %2;"
        : "=l"(state)
        : "r"(__as_ptr_smem(addr)),
          "r"(count)
        : "memory"
      );
    }
    return state;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_drop_is_only_supported_on_SM_90_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// mbarrier.arrive_drop.sem.scope.space.b64 _, [addr], count; // PTX ISA 80, SM_90
// .sem       = { .release }
// .scope     = { .cluster }
// .space     = { .shared::cluster }
template <typename = void>
__device__ static inline void mbarrier_arrive_drop(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  std::uint32_t count);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_drop_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void mbarrier_arrive_drop(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  std::uint32_t count)
{
  // sem == sem_release (due to parameter type constraint)
  // scope == scope_cluster (due to parameter type constraint)
  // space == space_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "mbarrier.arrive_drop.release.cluster.shared::cluster.b64 _, [%0], %1;"
      :
      : "r"(__as_ptr_remote_dsmem(addr)),
        "r"(count)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_drop_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// mbarrier.arrive_drop.sem.scope.space.b64 state, [addr], count; // PTX ISA 86, SM_90
// .sem       = { .relaxed }
// .scope     = { .cta, .cluster }
// .space     = { .shared::cta }
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive_drop(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  std::uint32_t count);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_drop_is_only_supported_on_SM_90_depending_on_the_variant__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive_drop(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  std::uint32_t count)
{
  // sem == sem_relaxed (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t state;
    if constexpr (scope == scope_cta) {
      asm (
        "mbarrier.arrive_drop.relaxed.cta.shared::cta.b64 %0, [%1], %2;"
        : "=l"(state)
        : "r"(__as_ptr_smem(addr)),
          "r"(count)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "mbarrier.arrive_drop.relaxed.cluster.shared::cta.b64 %0, [%1], %2;"
        : "=l"(state)
        : "r"(__as_ptr_smem(addr)),
          "r"(count)
        : "memory"
      );
    }
    return state;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_drop_is_only_supported_on_SM_90_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// mbarrier.arrive_drop.sem.scope.space.b64 _, [addr], count; // PTX ISA 86, SM_90
// .sem       = { .relaxed }
// .scope     = { .cluster }
// .space     = { .shared::cluster }
template <typename = void>
__device__ static inline void mbarrier_arrive_drop(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  std::uint32_t count);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_drop_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void mbarrier_arrive_drop(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  std::uint32_t count)
{
  // sem == sem_relaxed (due to parameter type constraint)
  // scope == scope_cluster (due to parameter type constraint)
  // space == space_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "mbarrier.arrive_drop.relaxed.cluster.shared::cluster.b64 _, [%0], %1;"
      :
      : "r"(__as_ptr_remote_dsmem(addr)),
        "r"(count)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_drop_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// mbarrier.arrive_drop.noComplete.release.cta.shared::cta.b64 state, [addr], count; // PTX ISA 80, SM_80
template <typename = void>
__device__ static inline std::uint64_t mbarrier_arrive_drop_no_complete(
  std::uint64_t* addr,
  std::uint32_t count);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_drop_no_complete_is_not_supported_before_SM_80__();
template <typename = void>
__device__ static inline std::uint64_t mbarrier_arrive_drop_no_complete(
  std::uint64_t* addr,
  std::uint32_t count)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint64_t state;
    asm (
      "mbarrier.arrive_drop.noComplete.release.cta.shared::cta.b64 %0, [%1], %2;"
      : "=l"(state)
      : "r"(__as_ptr_smem(addr)),
        "r"(count)
      : "memory"
    );
    return state;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_drop_no_complete_is_not_supported_before_SM_80__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// mbarrier.arrive_drop.expect_tx.sem.scope.space.b64 state, [addr], tx_count; // PTX ISA 80, SM_90
// .sem       = { .release }
// .scope     = { .cta, .cluster }
// .space     = { .shared::cta }
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive_drop_expect_tx(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  std::uint32_t tx_count);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_drop_expect_tx_is_only_supported_on_SM_90_depending_on_the_variant__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive_drop_expect_tx(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  std::uint32_t tx_count)
{
  // sem == sem_release (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t state;
    if constexpr (scope == scope_cta) {
      asm (
        "mbarrier.arrive_drop.expect_tx.release.cta.shared::cta.b64 %0, [%1], %2;"
        : "=l"(state)
        : "r"(__as_ptr_smem(addr)),
          "r"(tx_count)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "mbarrier.arrive_drop.expect_tx.release.cluster.shared::cta.b64 %0, [%1], %2;"
        : "=l"(state)
        : "r"(__as_ptr_smem(addr)),
          "r"(tx_count)
        : "memory"
      );
    }
    return state;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_drop_expect_tx_is_only_supported_on_SM_90_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// mbarrier.arrive_drop.expect_tx.sem.scope.space.b64 _, [addr], tx_count; // PTX ISA 80, SM_90
// .sem       = { .release }
// .scope     = { .cluster }
// .space     = { .shared::cluster }
template <typename = void>
__device__ static inline void mbarrier_arrive_drop_expect_tx(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  std::uint32_t tx_count);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_drop_expect_tx_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void mbarrier_arrive_drop_expect_tx(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  std::uint32_t tx_count)
{
  // sem == sem_release (due to parameter type constraint)
  // scope == scope_cluster (due to parameter type constraint)
  // space == space_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "mbarrier.arrive_drop.expect_tx.release.cluster.shared::cluster.b64 _, [%0], %1;"
      :
      : "r"(__as_ptr_remote_dsmem(addr)),
        "r"(tx_count)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_drop_expect_tx_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// mbarrier.arrive_drop.expect_tx.sem.scope.space.b64 state, [addr], tx_count; // PTX ISA 86, SM_90
// .sem       = { .relaxed }
// .scope     = { .cta, .cluster }
// .space     = { .shared::cta }
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive_drop_expect_tx(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  std::uint32_t tx_count);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_drop_expect_tx_is_only_supported_on_SM_90_depending_on_the_variant__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t mbarrier_arrive_drop_expect_tx(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  std::uint32_t tx_count)
{
  // sem == sem_relaxed (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t state;
    if constexpr (scope == scope_cta) {
      asm (
        "mbarrier.arrive_drop.expect_tx.relaxed.cta.shared::cta.b64 %0, [%1], %2;"
        : "=l"(state)
        : "r"(__as_ptr_smem(addr)),
          "r"(tx_count)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "mbarrier.arrive_drop.expect_tx.relaxed.cluster.shared::cta.b64 %0, [%1], %2;"
        : "=l"(state)
        : "r"(__as_ptr_smem(addr)),
          "r"(tx_count)
        : "memory"
      );
    }
    return state;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_drop_expect_tx_is_only_supported_on_SM_90_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// mbarrier.arrive_drop.expect_tx.sem.scope.space.b64 _, [addr], tx_count; // PTX ISA 86, SM_90
// .sem       = { .relaxed }
// .scope     = { .cluster }
// .space     = { .shared::cluster }
template <typename = void>
__device__ static inline void mbarrier_arrive_drop_expect_tx(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  std::uint32_t tx_count);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_mbarrier_arrive_drop_expect_tx_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void mbarrier_arrive_drop_expect_tx(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_cluster_t,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  std::uint32_t tx_count)
{
  // sem == sem_relaxed (due to parameter type constraint)
  // scope == scope_cluster (due to parameter type constraint)
  // space == space_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "mbarrier.arrive_drop.expect_tx.relaxed.cluster.shared::cluster.b64 _, [%0], %1;"
      :
      : "r"(__as_ptr_remote_dsmem(addr)),
        "r"(tx_count)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_arrive_drop_expect_tx_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

} // namespace cuda_ptx

#endif // INSTRUCTIONS_MBARRIER_ARRIVE_DROP_H

// End of ./instructions/mbarrier_arrive_drop.h
// Start of ./instructions/mbarrier_complete_tx.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_MBARRIER_COMPLETE_TX_H
#define INSTRUCTIONS_MBARRIER_COMPLETE_TX_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/mbarrier_complete_tx.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// mbarrier.complete_tx.sem.scope.space.b64 [addr], txCount; // PTX ISA 80, SM_90
// .sem       = { .relaxed }
// .scope     = { .cta, .cluster }
// .space     = { .shared::cta }
template <cuda_ptx::dot_scope Scope>
__device__ static inline void mbarrier_complete_tx(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  std::uint32_t txCount);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_complete_tx_is_only_supported_on_SM_90_depending_on_the_variant__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline void mbarrier_complete_tx(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  std::uint32_t txCount)
{
  // sem == sem_relaxed (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (scope == scope_cta) {
      asm (
        "mbarrier.complete_tx.relaxed.cta.shared::cta.b64 [%0], %1;"
        :
        : "r"(__as_ptr_smem(addr)),
          "r"(txCount)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "mbarrier.complete_tx.relaxed.cluster.shared::cta.b64 [%0], %1;"
        :
        : "r"(__as_ptr_smem(addr)),
          "r"(txCount)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_complete_tx_is_only_supported_on_SM_90_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// mbarrier.complete_tx.sem.scope.space.b64 [addr], txCount; // PTX ISA 80, SM_90
// .sem       = { .relaxed }
// .scope     = { .cta, .cluster }
// .space     = { .shared::cluster }
template <cuda_ptx::dot_scope Scope>
__device__ static inline void mbarrier_complete_tx(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  std::uint32_t txCount);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_complete_tx_is_only_supported_on_SM_90_depending_on_the_variant__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline void mbarrier_complete_tx(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  std::uint32_t txCount)
{
  // sem == sem_relaxed (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  // space == space_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (scope == scope_cta) {
      asm (
        "mbarrier.complete_tx.relaxed.cta.shared::cluster.b64 [%0], %1;"
        :
        : "r"(__as_ptr_dsmem(addr)),
          "r"(txCount)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "mbarrier.complete_tx.relaxed.cluster.shared::cluster.b64 [%0], %1;"
        :
        : "r"(__as_ptr_dsmem(addr)),
          "r"(txCount)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_complete_tx_is_only_supported_on_SM_90_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

} // namespace cuda_ptx

#endif // INSTRUCTIONS_MBARRIER_COMPLETE_TX_H

// End of ./instructions/mbarrier_complete_tx.h
// Start of ./instructions/mbarrier_expect_tx.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_MBARRIER_EXPECT_TX_H
#define INSTRUCTIONS_MBARRIER_EXPECT_TX_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/mbarrier_expect_tx.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// mbarrier.expect_tx.sem.scope.space.b64 [addr], txCount; // 1. PTX ISA 80, SM_90
// .sem       = { .relaxed }
// .scope     = { .cta, .cluster }
// .space     = { .shared::cta }
template <cuda_ptx::dot_scope Scope>
__device__ static inline void mbarrier_expect_tx(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  std::uint32_t txCount);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_expect_tx_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline void mbarrier_expect_tx(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_shared_t,
  std::uint64_t* addr,
  std::uint32_t txCount)
{
  // sem == sem_relaxed (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (scope == scope_cta) {
      asm (
        "mbarrier.expect_tx.relaxed.cta.shared::cta.b64 [%0], %1; // 1."
        :
        : "r"(__as_ptr_smem(addr)),
          "r"(txCount)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "mbarrier.expect_tx.relaxed.cluster.shared::cta.b64 [%0], %1; // 1."
        :
        : "r"(__as_ptr_smem(addr)),
          "r"(txCount)
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_expect_tx_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// mbarrier.expect_tx.sem.scope.space.b64 [addr], txCount; // 2. PTX ISA 80, SM_90
// .sem       = { .relaxed }
// .scope     = { .cta, .cluster }
// .space     = { .shared::cluster }
template <cuda_ptx::dot_scope Scope>
__device__ static inline void mbarrier_expect_tx(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  std::uint32_t txCount);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_expect_tx_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline void mbarrier_expect_tx(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::space_cluster_t,
  std::uint64_t* addr,
  std::uint32_t txCount)
{
  // sem == sem_relaxed (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  // space == space_cluster (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (scope == scope_cta) {
      asm (
        "mbarrier.expect_tx.relaxed.cta.shared::cluster.b64 [%0], %1; // 2."
        :
        : "r"(__as_ptr_dsmem(addr)),
          "r"(txCount)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "mbarrier.expect_tx.relaxed.cluster.shared::cluster.b64 [%0], %1; // 2."
        :
        : "r"(__as_ptr_dsmem(addr)),
          "r"(txCount)
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_expect_tx_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

} // namespace cuda_ptx

#endif // INSTRUCTIONS_MBARRIER_EXPECT_TX_H

// End of ./instructions/mbarrier_expect_tx.h
// Start of ./instructions/mbarrier_init.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_MBARRIER_INIT_H
#define INSTRUCTIONS_MBARRIER_INIT_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/mbarrier_init.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// mbarrier.init.shared.b64 [addr], count; // PTX ISA 70, SM_80
template <typename = void>
__device__ static inline void mbarrier_init(
  std::uint64_t* addr,
  const std::uint32_t& count);
*/
#if __libcuda_ptx_isa >= 700
extern "C" __device__ void __cuda_ptx_mbarrier_init_is_not_supported_before_SM_80__();
template <typename = void>
__device__ static inline void mbarrier_init(
  std::uint64_t* addr,
  const std::uint32_t& count)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "mbarrier.init.shared.b64 [%0], %1;"
      :
      : "r"(__as_ptr_smem(addr)),
        "r"(count)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_init_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 700

} // namespace cuda_ptx

#endif // INSTRUCTIONS_MBARRIER_INIT_H

// End of ./instructions/mbarrier_init.h
// Start of ./instructions/mbarrier_inval.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_MBARRIER_INVAL_H
#define INSTRUCTIONS_MBARRIER_INVAL_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/mbarrier_inval.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// mbarrier.inval.shared.b64 [addr]; // PTX ISA 70, SM_80
template <typename = void>
__device__ static inline void mbarrier_inval(
  std::uint64_t* addr);
*/
#if __libcuda_ptx_isa >= 700
extern "C" __device__ void __cuda_ptx_mbarrier_inval_is_not_supported_before_SM_80__();
template <typename = void>
__device__ static inline void mbarrier_inval(
  std::uint64_t* addr)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "mbarrier.inval.shared.b64 [%0];"
      :
      : "r"(__as_ptr_smem(addr))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_inval_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 700

} // namespace cuda_ptx

#endif // INSTRUCTIONS_MBARRIER_INVAL_H

// End of ./instructions/mbarrier_inval.h
// Start of ./instructions/mbarrier_pending_count.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_MBARRIER_PENDING_COUNT_H
#define INSTRUCTIONS_MBARRIER_PENDING_COUNT_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/mbarrier_pending_count.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// mbarrier.pending_count.b64 count, state; // PTX ISA 70, SM_80
template <typename = void>
__device__ static inline std::uint32_t mbarrier_pending_count(
  std::uint64_t state);
*/
#if __libcuda_ptx_isa >= 700
extern "C" __device__ void __cuda_ptx_mbarrier_pending_count_is_not_supported_before_SM_80__();
template <typename = void>
__device__ static inline std::uint32_t mbarrier_pending_count(
  std::uint64_t state)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t count;
    asm volatile (
      "mbarrier.pending_count.b64 %0, %1;"
      : "=r"(count)
      : "l"(state)
      :
    );
    return count;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_pending_count_is_not_supported_before_SM_80__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 700

} // namespace cuda_ptx

#endif // INSTRUCTIONS_MBARRIER_PENDING_COUNT_H

// End of ./instructions/mbarrier_pending_count.h
// Start of ./instructions/mbarrier_wait.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_MBARRIER_WAIT_H
#define INSTRUCTIONS_MBARRIER_WAIT_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/mbarrier_wait.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

#if defined(__NVCC__)
#pragma nv_diag_suppress 549 // "parameter is never used"
#endif

namespace cuda_ptx {

/*
// mbarrier.test_wait.shared.b64 waitComplete, [addr], state; // PTX ISA 70, SM_80
template <typename = void>
__device__ static inline bool mbarrier_test_wait(
  std::uint64_t* addr,
  const std::uint64_t& state);
*/
#if __libcuda_ptx_isa >= 700
extern "C" __device__ void __cuda_ptx_mbarrier_test_wait_is_not_supported_before_SM_80__();
template <typename = void>
__device__ static inline bool mbarrier_test_wait(
  std::uint64_t* addr,
  const std::uint64_t& state)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t waitComplete;
    asm (
      "{\n\t"
      ".reg .pred P_OUT; \n\t"
      "mbarrier.test_wait.shared.b64 P_OUT, [%1], %2; \n\t"
      "selp.b32 %0, 1, 0, P_OUT; \n"
      "}"
      : "=r"(waitComplete)
      : "r"(__as_ptr_smem(addr)),
        "l"(state)
      : "memory"
    );
    return static_cast<bool>(waitComplete);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_test_wait_is_not_supported_before_SM_80__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 700

/*
// mbarrier.test_wait.sem.scope.shared::cta.b64 waitComplete, [addr], state; // PTX ISA 80, SM_90
// .sem       = { .acquire }
// .scope     = { .cta, .cluster }
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_test_wait(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint64_t& state);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_test_wait_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_test_wait(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint64_t& state)
{
  // sem == sem_acquire (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t waitComplete;
    if constexpr (scope == scope_cta) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.test_wait.acquire.cta.shared::cta.b64 P_OUT, [%1], %2; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "l"(state)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.test_wait.acquire.cluster.shared::cta.b64 P_OUT, [%1], %2; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "l"(state)
        : "memory"
      );
    }
    return static_cast<bool>(waitComplete);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_test_wait_is_not_supported_before_SM_90__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// mbarrier.test_wait.sem.scope.shared::cta.b64 waitComplete, [addr], state; // PTX ISA 86, SM_90
// .sem       = { .relaxed }
// .scope     = { .cta, .cluster }
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_test_wait(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint64_t& state);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_mbarrier_test_wait_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_test_wait(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint64_t& state)
{
  // sem == sem_relaxed (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t waitComplete;
    if constexpr (scope == scope_cta) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.test_wait.relaxed.cta.shared::cta.b64 P_OUT, [%1], %2; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "l"(state)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.test_wait.relaxed.cluster.shared::cta.b64 P_OUT, [%1], %2; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "l"(state)
        : "memory"
      );
    }
    return static_cast<bool>(waitComplete);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_test_wait_is_not_supported_before_SM_90__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 860
/*
// mbarrier.test_wait.parity.shared.b64 waitComplete, [addr], phaseParity; // PTX ISA 71, SM_80
template <typename = void>
__device__ static inline bool mbarrier_test_wait_parity(
  std::uint64_t* addr,
  const std::uint32_t& phaseParity);
*/
#if __libcuda_ptx_isa >= 710
extern "C" __device__ void __cuda_ptx_mbarrier_test_wait_parity_is_not_supported_before_SM_80__();
template <typename = void>
__device__ static inline bool mbarrier_test_wait_parity(
  std::uint64_t* addr,
  const std::uint32_t& phaseParity)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    std::uint32_t waitComplete;
    asm (
      "{\n\t"
      ".reg .pred P_OUT; \n\t"
      "mbarrier.test_wait.parity.shared.b64 P_OUT, [%1], %2; \n\t"
      "selp.b32 %0, 1, 0, P_OUT; \n"
      "}"
      : "=r"(waitComplete)
      : "r"(__as_ptr_smem(addr)),
        "r"(phaseParity)
      : "memory"
    );
    return static_cast<bool>(waitComplete);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_test_wait_parity_is_not_supported_before_SM_80__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 710

/*
// mbarrier.test_wait.parity.sem.scope.shared::cta.b64 waitComplete, [addr], phaseParity; // PTX ISA 80, SM_90
// .sem       = { .acquire }
// .scope     = { .cta, .cluster }
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_test_wait_parity(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint32_t& phaseParity);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_test_wait_parity_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_test_wait_parity(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint32_t& phaseParity)
{
  // sem == sem_acquire (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t waitComplete;
    if constexpr (scope == scope_cta) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.test_wait.parity.acquire.cta.shared::cta.b64 P_OUT, [%1], %2; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "r"(phaseParity)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.test_wait.parity.acquire.cluster.shared::cta.b64 P_OUT, [%1], %2; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "r"(phaseParity)
        : "memory"
      );
    }
    return static_cast<bool>(waitComplete);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_test_wait_parity_is_not_supported_before_SM_90__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// mbarrier.test_wait.parity.sem.scope.shared::cta.b64 waitComplete, [addr], phaseParity; // PTX ISA 86, SM_90
// .sem       = { .relaxed }
// .scope     = { .cta, .cluster }
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_test_wait_parity(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint32_t& phaseParity);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_mbarrier_test_wait_parity_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_test_wait_parity(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint32_t& phaseParity)
{
  // sem == sem_relaxed (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t waitComplete;
    if constexpr (scope == scope_cta) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.test_wait.parity.relaxed.cta.shared::cta.b64 P_OUT, [%1], %2; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "r"(phaseParity)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.test_wait.parity.relaxed.cluster.shared::cta.b64 P_OUT, [%1], %2; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "r"(phaseParity)
        : "memory"
      );
    }
    return static_cast<bool>(waitComplete);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_test_wait_parity_is_not_supported_before_SM_90__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// mbarrier.try_wait.shared::cta.b64 waitComplete, [addr], state; // PTX ISA 78, SM_90
template <typename = void>
__device__ static inline bool mbarrier_try_wait(
  std::uint64_t* addr,
  const std::uint64_t& state);
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline bool mbarrier_try_wait(
  std::uint64_t* addr,
  const std::uint64_t& state)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t waitComplete;
    asm (
      "{\n\t"
      ".reg .pred P_OUT; \n\t"
      "mbarrier.try_wait.shared::cta.b64 P_OUT, [%1], %2; \n\t"
      "selp.b32 %0, 1, 0, P_OUT; \n"
      "}"
      : "=r"(waitComplete)
      : "r"(__as_ptr_smem(addr)),
        "l"(state)
      : "memory"
    );
    return static_cast<bool>(waitComplete);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mbarrier.try_wait.shared::cta.b64 waitComplete, [addr], state, suspendTimeHint; // PTX ISA 78, SM_90
template <typename = void>
__device__ static inline bool mbarrier_try_wait(
  std::uint64_t* addr,
  const std::uint64_t& state,
  const std::uint32_t& suspendTimeHint);
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline bool mbarrier_try_wait(
  std::uint64_t* addr,
  const std::uint64_t& state,
  const std::uint32_t& suspendTimeHint)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t waitComplete;
    asm (
      "{\n\t"
      ".reg .pred P_OUT; \n\t"
      "mbarrier.try_wait.shared::cta.b64 P_OUT, [%1], %2, %3; \n\t"
      "selp.b32 %0, 1, 0, P_OUT; \n"
      "}"
      : "=r"(waitComplete)
      : "r"(__as_ptr_smem(addr)),
        "l"(state),
        "r"(suspendTimeHint)
      : "memory"
    );
    return static_cast<bool>(waitComplete);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mbarrier.try_wait.sem.scope.shared::cta.b64 waitComplete, [addr], state; // PTX ISA 80, SM_90
// .sem       = { .acquire }
// .scope     = { .cta, .cluster }
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint64_t& state);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint64_t& state)
{
  // sem == sem_acquire (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t waitComplete;
    if constexpr (scope == scope_cta) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.try_wait.acquire.cta.shared::cta.b64 P_OUT, [%1], %2; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "l"(state)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.try_wait.acquire.cluster.shared::cta.b64 P_OUT, [%1], %2; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "l"(state)
        : "memory"
      );
    }
    return static_cast<bool>(waitComplete);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// mbarrier.try_wait.sem.scope.shared::cta.b64 waitComplete, [addr], state, suspendTimeHint; // PTX ISA 80, SM_90
// .sem       = { .acquire }
// .scope     = { .cta, .cluster }
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint64_t& state,
  const std::uint32_t& suspendTimeHint);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint64_t& state,
  const std::uint32_t& suspendTimeHint)
{
  // sem == sem_acquire (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t waitComplete;
    if constexpr (scope == scope_cta) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.try_wait.acquire.cta.shared::cta.b64 P_OUT, [%1], %2, %3; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "l"(state),
          "r"(suspendTimeHint)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.try_wait.acquire.cluster.shared::cta.b64 P_OUT, [%1], %2, %3; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "l"(state),
          "r"(suspendTimeHint)
        : "memory"
      );
    }
    return static_cast<bool>(waitComplete);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// mbarrier.try_wait.sem.scope.shared::cta.b64 waitComplete, [addr], state; // PTX ISA 86, SM_90
// .sem       = { .relaxed }
// .scope     = { .cta, .cluster }
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint64_t& state);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint64_t& state)
{
  // sem == sem_relaxed (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t waitComplete;
    if constexpr (scope == scope_cta) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.try_wait.relaxed.cta.shared::cta.b64 P_OUT, [%1], %2; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "l"(state)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.try_wait.relaxed.cluster.shared::cta.b64 P_OUT, [%1], %2; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "l"(state)
        : "memory"
      );
    }
    return static_cast<bool>(waitComplete);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// mbarrier.try_wait.sem.scope.shared::cta.b64 waitComplete, [addr], state, suspendTimeHint; // PTX ISA 86, SM_90
// .sem       = { .relaxed }
// .scope     = { .cta, .cluster }
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint64_t& state,
  const std::uint32_t& suspendTimeHint);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint64_t& state,
  const std::uint32_t& suspendTimeHint)
{
  // sem == sem_relaxed (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t waitComplete;
    if constexpr (scope == scope_cta) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.try_wait.relaxed.cta.shared::cta.b64 P_OUT, [%1], %2, %3; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "l"(state),
          "r"(suspendTimeHint)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.try_wait.relaxed.cluster.shared::cta.b64 P_OUT, [%1], %2, %3; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "l"(state),
          "r"(suspendTimeHint)
        : "memory"
      );
    }
    return static_cast<bool>(waitComplete);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_try_wait_is_not_supported_before_SM_90__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 860
/*
// mbarrier.try_wait.parity.shared::cta.b64 waitComplete, [addr], phaseParity; // PTX ISA 78, SM_90
template <typename = void>
__device__ static inline bool mbarrier_try_wait_parity(
  std::uint64_t* addr,
  const std::uint32_t& phaseParity);
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline bool mbarrier_try_wait_parity(
  std::uint64_t* addr,
  const std::uint32_t& phaseParity)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t waitComplete;
    asm (
      "{\n\t"
      ".reg .pred P_OUT; \n\t"
      "mbarrier.try_wait.parity.shared::cta.b64 P_OUT, [%1], %2; \n\t"
      "selp.b32 %0, 1, 0, P_OUT; \n"
      "}"
      : "=r"(waitComplete)
      : "r"(__as_ptr_smem(addr)),
        "r"(phaseParity)
      : "memory"
    );
    return static_cast<bool>(waitComplete);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mbarrier.try_wait.parity.shared::cta.b64 waitComplete, [addr], phaseParity, suspendTimeHint; // PTX ISA 78, SM_90
template <typename = void>
__device__ static inline bool mbarrier_try_wait_parity(
  std::uint64_t* addr,
  const std::uint32_t& phaseParity,
  const std::uint32_t& suspendTimeHint);
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline bool mbarrier_try_wait_parity(
  std::uint64_t* addr,
  const std::uint32_t& phaseParity,
  const std::uint32_t& suspendTimeHint)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t waitComplete;
    asm (
      "{\n\t"
      ".reg .pred P_OUT; \n\t"
      "mbarrier.try_wait.parity.shared::cta.b64 P_OUT, [%1], %2, %3; \n\t"
      "selp.b32 %0, 1, 0, P_OUT; \n"
      "}"
      : "=r"(waitComplete)
      : "r"(__as_ptr_smem(addr)),
        "r"(phaseParity),
        "r"(suspendTimeHint)
      : "memory"
    );
    return static_cast<bool>(waitComplete);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// mbarrier.try_wait.parity.sem.scope.shared::cta.b64 waitComplete, [addr], phaseParity; // PTX ISA 80, SM_90
// .sem       = { .acquire }
// .scope     = { .cta, .cluster }
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait_parity(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint32_t& phaseParity);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait_parity(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint32_t& phaseParity)
{
  // sem == sem_acquire (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t waitComplete;
    if constexpr (scope == scope_cta) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.try_wait.parity.acquire.cta.shared::cta.b64 P_OUT, [%1], %2; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "r"(phaseParity)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.try_wait.parity.acquire.cluster.shared::cta.b64 P_OUT, [%1], %2; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "r"(phaseParity)
        : "memory"
      );
    }
    return static_cast<bool>(waitComplete);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// mbarrier.try_wait.parity.sem.scope.shared::cta.b64 waitComplete, [addr], phaseParity, suspendTimeHint; // PTX ISA 80, SM_90
// .sem       = { .acquire }
// .scope     = { .cta, .cluster }
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait_parity(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint32_t& phaseParity,
  const std::uint32_t& suspendTimeHint);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait_parity(
  cuda_ptx::sem_acquire_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint32_t& phaseParity,
  const std::uint32_t& suspendTimeHint)
{
  // sem == sem_acquire (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t waitComplete;
    if constexpr (scope == scope_cta) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.try_wait.parity.acquire.cta.shared::cta.b64 P_OUT, [%1], %2, %3; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "r"(phaseParity),
          "r"(suspendTimeHint)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.try_wait.parity.acquire.cluster.shared::cta.b64 P_OUT, [%1], %2, %3; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "r"(phaseParity),
          "r"(suspendTimeHint)
        : "memory"
      );
    }
    return static_cast<bool>(waitComplete);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// mbarrier.try_wait.parity.sem.scope.shared::cta.b64 waitComplete, [addr], phaseParity; // PTX ISA 86, SM_90
// .sem       = { .relaxed }
// .scope     = { .cta, .cluster }
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait_parity(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint32_t& phaseParity);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait_parity(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint32_t& phaseParity)
{
  // sem == sem_relaxed (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t waitComplete;
    if constexpr (scope == scope_cta) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.try_wait.parity.relaxed.cta.shared::cta.b64 P_OUT, [%1], %2; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "r"(phaseParity)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.try_wait.parity.relaxed.cluster.shared::cta.b64 P_OUT, [%1], %2; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "r"(phaseParity)
        : "memory"
      );
    }
    return static_cast<bool>(waitComplete);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// mbarrier.try_wait.parity.sem.scope.shared::cta.b64 waitComplete, [addr], phaseParity, suspendTimeHint; // PTX ISA 86, SM_90
// .sem       = { .relaxed }
// .scope     = { .cta, .cluster }
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait_parity(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint32_t& phaseParity,
  const std::uint32_t& suspendTimeHint);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_scope Scope>
__device__ static inline bool mbarrier_try_wait_parity(
  cuda_ptx::sem_relaxed_t,
  cuda_ptx::scope_t<Scope> scope,
  std::uint64_t* addr,
  const std::uint32_t& phaseParity,
  const std::uint32_t& suspendTimeHint)
{
  // sem == sem_relaxed (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t waitComplete;
    if constexpr (scope == scope_cta) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.try_wait.parity.relaxed.cta.shared::cta.b64 P_OUT, [%1], %2, %3; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "r"(phaseParity),
          "r"(suspendTimeHint)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm (
        "{\n\t"
        ".reg .pred P_OUT; \n\t"
        "mbarrier.try_wait.parity.relaxed.cluster.shared::cta.b64 P_OUT, [%1], %2, %3; \n\t"
        "selp.b32 %0, 1, 0, P_OUT; \n"
        "}"
        : "=r"(waitComplete)
        : "r"(__as_ptr_smem(addr)),
          "r"(phaseParity),
          "r"(suspendTimeHint)
        : "memory"
      );
    }
    return static_cast<bool>(waitComplete);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_mbarrier_try_wait_parity_is_not_supported_before_SM_90__();
    return false;
  #endif
}
#endif // __libcuda_ptx_isa >= 860

} // namespace cuda_ptx

#if defined(__NVCC__)
#pragma nv_diag_default 549
#endif

#endif // INSTRUCTIONS_MBARRIER_WAIT_H

// End of ./instructions/mbarrier_wait.h
// Start of ./instructions/multimem.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_MULTIMEM_H
#define INSTRUCTIONS_MULTIMEM_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/multimem.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// multimem.ld_reduce.sem.global.op.u32 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .weak }
// .op        = { .min }
template <typename = void>
__device__ static inline std::uint32_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_min_t,
  const std::uint32_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint32_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_min_t,
  const std::uint32_t* addr)
{
  // sem == sem_weak (due to parameter type constraint)
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t dest;
    asm (
      "multimem.ld_reduce.weak.global.min.u32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.scope.global.op.u32 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .min }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::uint32_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_min_t,
  const std::uint32_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::uint32_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_min_t,
  const std::uint32_t* addr)
{
  static_assert(sem == sem_relaxed || sem == sem_acquire, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t dest;
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.relaxed.cta.global.min.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.relaxed.cluster.global.min.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.relaxed.gpu.global.min.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.relaxed.sys.global.min.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.acquire.cta.global.min.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.acquire.cluster.global.min.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.acquire.gpu.global.min.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.acquire.sys.global.min.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    }
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.global.op.u64 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .weak }
// .op        = { .min }
template <typename = void>
__device__ static inline std::uint64_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_min_t,
  const std::uint64_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint64_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_min_t,
  const std::uint64_t* addr)
{
  // sem == sem_weak (due to parameter type constraint)
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t dest;
    asm (
      "multimem.ld_reduce.weak.global.min.u64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.scope.global.op.u64 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .min }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_min_t,
  const std::uint64_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_min_t,
  const std::uint64_t* addr)
{
  static_assert(sem == sem_relaxed || sem == sem_acquire, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t dest;
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.relaxed.cta.global.min.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.relaxed.cluster.global.min.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.relaxed.gpu.global.min.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.relaxed.sys.global.min.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.acquire.cta.global.min.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.acquire.cluster.global.min.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.acquire.gpu.global.min.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.acquire.sys.global.min.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    }
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.global.op.s32 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .weak }
// .op        = { .min }
template <typename = void>
__device__ static inline std::int32_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_min_t,
  const std::int32_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::int32_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_min_t,
  const std::int32_t* addr)
{
  // sem == sem_weak (due to parameter type constraint)
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::int32_t dest;
    asm (
      "multimem.ld_reduce.weak.global.min.s32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.scope.global.op.s32 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .min }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::int32_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_min_t,
  const std::int32_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::int32_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_min_t,
  const std::int32_t* addr)
{
  static_assert(sem == sem_relaxed || sem == sem_acquire, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::int32_t dest;
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.relaxed.cta.global.min.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.relaxed.cluster.global.min.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.relaxed.gpu.global.min.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.relaxed.sys.global.min.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.acquire.cta.global.min.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.acquire.cluster.global.min.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.acquire.gpu.global.min.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.acquire.sys.global.min.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    }
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.global.op.s64 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .weak }
// .op        = { .min }
template <typename = void>
__device__ static inline std::int64_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_min_t,
  const std::int64_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::int64_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_min_t,
  const std::int64_t* addr)
{
  // sem == sem_weak (due to parameter type constraint)
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::int64_t dest;
    asm (
      "multimem.ld_reduce.weak.global.min.s64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.scope.global.op.s64 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .min }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::int64_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_min_t,
  const std::int64_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::int64_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_min_t,
  const std::int64_t* addr)
{
  static_assert(sem == sem_relaxed || sem == sem_acquire, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::int64_t dest;
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.relaxed.cta.global.min.s64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.relaxed.cluster.global.min.s64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.relaxed.gpu.global.min.s64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.relaxed.sys.global.min.s64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.acquire.cta.global.min.s64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.acquire.cluster.global.min.s64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.acquire.gpu.global.min.s64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.acquire.sys.global.min.s64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    }
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.global.op.u32 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .weak }
// .op        = { .max }
template <typename = void>
__device__ static inline std::uint32_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_max_t,
  const std::uint32_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint32_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_max_t,
  const std::uint32_t* addr)
{
  // sem == sem_weak (due to parameter type constraint)
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t dest;
    asm (
      "multimem.ld_reduce.weak.global.max.u32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.scope.global.op.u32 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .max }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::uint32_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_max_t,
  const std::uint32_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::uint32_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_max_t,
  const std::uint32_t* addr)
{
  static_assert(sem == sem_relaxed || sem == sem_acquire, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t dest;
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.relaxed.cta.global.max.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.relaxed.cluster.global.max.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.relaxed.gpu.global.max.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.relaxed.sys.global.max.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.acquire.cta.global.max.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.acquire.cluster.global.max.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.acquire.gpu.global.max.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.acquire.sys.global.max.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    }
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.global.op.u64 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .weak }
// .op        = { .max }
template <typename = void>
__device__ static inline std::uint64_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_max_t,
  const std::uint64_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint64_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_max_t,
  const std::uint64_t* addr)
{
  // sem == sem_weak (due to parameter type constraint)
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t dest;
    asm (
      "multimem.ld_reduce.weak.global.max.u64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.scope.global.op.u64 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .max }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_max_t,
  const std::uint64_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_max_t,
  const std::uint64_t* addr)
{
  static_assert(sem == sem_relaxed || sem == sem_acquire, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t dest;
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.relaxed.cta.global.max.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.relaxed.cluster.global.max.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.relaxed.gpu.global.max.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.relaxed.sys.global.max.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.acquire.cta.global.max.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.acquire.cluster.global.max.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.acquire.gpu.global.max.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.acquire.sys.global.max.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    }
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.global.op.s32 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .weak }
// .op        = { .max }
template <typename = void>
__device__ static inline std::int32_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_max_t,
  const std::int32_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::int32_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_max_t,
  const std::int32_t* addr)
{
  // sem == sem_weak (due to parameter type constraint)
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::int32_t dest;
    asm (
      "multimem.ld_reduce.weak.global.max.s32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.scope.global.op.s32 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .max }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::int32_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_max_t,
  const std::int32_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::int32_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_max_t,
  const std::int32_t* addr)
{
  static_assert(sem == sem_relaxed || sem == sem_acquire, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::int32_t dest;
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.relaxed.cta.global.max.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.relaxed.cluster.global.max.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.relaxed.gpu.global.max.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.relaxed.sys.global.max.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.acquire.cta.global.max.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.acquire.cluster.global.max.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.acquire.gpu.global.max.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.acquire.sys.global.max.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    }
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.global.op.s64 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .weak }
// .op        = { .max }
template <typename = void>
__device__ static inline std::int64_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_max_t,
  const std::int64_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::int64_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_max_t,
  const std::int64_t* addr)
{
  // sem == sem_weak (due to parameter type constraint)
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::int64_t dest;
    asm (
      "multimem.ld_reduce.weak.global.max.s64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.scope.global.op.s64 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .max }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::int64_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_max_t,
  const std::int64_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::int64_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_max_t,
  const std::int64_t* addr)
{
  static_assert(sem == sem_relaxed || sem == sem_acquire, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::int64_t dest;
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.relaxed.cta.global.max.s64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.relaxed.cluster.global.max.s64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.relaxed.gpu.global.max.s64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.relaxed.sys.global.max.s64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.acquire.cta.global.max.s64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.acquire.cluster.global.max.s64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.acquire.gpu.global.max.s64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.acquire.sys.global.max.s64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    }
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.global.op.u32 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .weak }
// .op        = { .add }
template <typename = void>
__device__ static inline std::uint32_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_add_t,
  const std::uint32_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint32_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_add_t,
  const std::uint32_t* addr)
{
  // sem == sem_weak (due to parameter type constraint)
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t dest;
    asm (
      "multimem.ld_reduce.weak.global.add.u32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.scope.global.op.u32 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .add }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::uint32_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_add_t,
  const std::uint32_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::uint32_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_add_t,
  const std::uint32_t* addr)
{
  static_assert(sem == sem_relaxed || sem == sem_acquire, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t dest;
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.relaxed.cta.global.add.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.relaxed.cluster.global.add.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.relaxed.gpu.global.add.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.relaxed.sys.global.add.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.acquire.cta.global.add.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.acquire.cluster.global.add.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.acquire.gpu.global.add.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.acquire.sys.global.add.u32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    }
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.global.op.u64 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .weak }
// .op        = { .add }
template <typename = void>
__device__ static inline std::uint64_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_add_t,
  const std::uint64_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::uint64_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_add_t,
  const std::uint64_t* addr)
{
  // sem == sem_weak (due to parameter type constraint)
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t dest;
    asm (
      "multimem.ld_reduce.weak.global.add.u64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.scope.global.op.u64 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .add }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_add_t,
  const std::uint64_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::uint64_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_add_t,
  const std::uint64_t* addr)
{
  static_assert(sem == sem_relaxed || sem == sem_acquire, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t dest;
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.relaxed.cta.global.add.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.relaxed.cluster.global.add.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.relaxed.gpu.global.add.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.relaxed.sys.global.add.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.acquire.cta.global.add.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.acquire.cluster.global.add.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.acquire.gpu.global.add.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.acquire.sys.global.add.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    }
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.global.op.s32 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .weak }
// .op        = { .add }
template <typename = void>
__device__ static inline std::int32_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_add_t,
  const std::int32_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::int32_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_add_t,
  const std::int32_t* addr)
{
  // sem == sem_weak (due to parameter type constraint)
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::int32_t dest;
    asm (
      "multimem.ld_reduce.weak.global.add.s32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.scope.global.op.s32 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .add }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::int32_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_add_t,
  const std::int32_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::int32_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_add_t,
  const std::int32_t* addr)
{
  static_assert(sem == sem_relaxed || sem == sem_acquire, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::int32_t dest;
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.relaxed.cta.global.add.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.relaxed.cluster.global.add.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.relaxed.gpu.global.add.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.relaxed.sys.global.add.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.acquire.cta.global.add.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.acquire.cluster.global.add.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.acquire.gpu.global.add.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.acquire.sys.global.add.s32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    }
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.global.op.u64 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .weak }
// .op        = { .add }
template <typename = void>
__device__ static inline std::int64_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_add_t,
  const std::int64_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline std::int64_t multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_add_t,
  const std::int64_t* addr)
{
  // sem == sem_weak (due to parameter type constraint)
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::int64_t dest;
    asm (
      "multimem.ld_reduce.weak.global.add.u64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.scope.global.op.u64 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .add }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::int64_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_add_t,
  const std::int64_t* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline std::int64_t multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_add_t,
  const std::int64_t* addr)
{
  static_assert(sem == sem_relaxed || sem == sem_acquire, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::int64_t dest;
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.relaxed.cta.global.add.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.relaxed.cluster.global.add.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.relaxed.gpu.global.add.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.relaxed.sys.global.add.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.acquire.cta.global.add.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.acquire.cluster.global.add.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.acquire.gpu.global.add.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.acquire.sys.global.add.u64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    }
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.global.op.b32 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .weak }
// .op        = { .and }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_and_op_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_and_op_t,
  const B32* addr)
{
  // sem == sem_weak (due to parameter type constraint)
  // op == op_and_op (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t dest;
    asm (
      "multimem.ld_reduce.weak.global.and.b32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.scope.global.op.b32 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .and }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline B32 multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_and_op_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline B32 multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_and_op_t,
  const B32* addr)
{
  static_assert(sem == sem_relaxed || sem == sem_acquire, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_and_op (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t dest;
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.relaxed.cta.global.and.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.relaxed.cluster.global.and.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.relaxed.gpu.global.and.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.relaxed.sys.global.and.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.acquire.cta.global.and.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.acquire.cluster.global.and.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.acquire.gpu.global.and.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.acquire.sys.global.and.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    }
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.global.op.b32 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .weak }
// .op        = { .or }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_or_op_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_or_op_t,
  const B32* addr)
{
  // sem == sem_weak (due to parameter type constraint)
  // op == op_or_op (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t dest;
    asm (
      "multimem.ld_reduce.weak.global.or.b32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.scope.global.op.b32 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .or }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline B32 multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_or_op_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline B32 multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_or_op_t,
  const B32* addr)
{
  static_assert(sem == sem_relaxed || sem == sem_acquire, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_or_op (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t dest;
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.relaxed.cta.global.or.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.relaxed.cluster.global.or.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.relaxed.gpu.global.or.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.relaxed.sys.global.or.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.acquire.cta.global.or.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.acquire.cluster.global.or.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.acquire.gpu.global.or.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.acquire.sys.global.or.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    }
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.global.op.b32 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .weak }
// .op        = { .xor }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_xor_op_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_xor_op_t,
  const B32* addr)
{
  // sem == sem_weak (due to parameter type constraint)
  // op == op_xor_op (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t dest;
    asm (
      "multimem.ld_reduce.weak.global.xor.b32 %0, [%1];"
      : "=r"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.scope.global.op.b32 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .xor }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline B32 multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_xor_op_t,
  const B32* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline B32 multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_xor_op_t,
  const B32* addr)
{
  static_assert(sem == sem_relaxed || sem == sem_acquire, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_xor_op (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint32_t dest;
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.relaxed.cta.global.xor.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.relaxed.cluster.global.xor.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.relaxed.gpu.global.xor.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.relaxed.sys.global.xor.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.acquire.cta.global.xor.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.acquire.cluster.global.xor.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.acquire.gpu.global.xor.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.acquire.sys.global.xor.b32 %0, [%1];"
        : "=r"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    }
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.global.op.b64 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .weak }
// .op        = { .and }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_and_op_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_and_op_t,
  const B64* addr)
{
  // sem == sem_weak (due to parameter type constraint)
  // op == op_and_op (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t dest;
    asm (
      "multimem.ld_reduce.weak.global.and.b64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.scope.global.op.b64 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .and }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline B64 multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_and_op_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline B64 multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_and_op_t,
  const B64* addr)
{
  static_assert(sem == sem_relaxed || sem == sem_acquire, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_and_op (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t dest;
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.relaxed.cta.global.and.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.relaxed.cluster.global.and.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.relaxed.gpu.global.and.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.relaxed.sys.global.and.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.acquire.cta.global.and.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.acquire.cluster.global.and.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.acquire.gpu.global.and.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.acquire.sys.global.and.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    }
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.global.op.b64 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .weak }
// .op        = { .or }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_or_op_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_or_op_t,
  const B64* addr)
{
  // sem == sem_weak (due to parameter type constraint)
  // op == op_or_op (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t dest;
    asm (
      "multimem.ld_reduce.weak.global.or.b64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.scope.global.op.b64 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .or }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline B64 multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_or_op_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline B64 multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_or_op_t,
  const B64* addr)
{
  static_assert(sem == sem_relaxed || sem == sem_acquire, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_or_op (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t dest;
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.relaxed.cta.global.or.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.relaxed.cluster.global.or.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.relaxed.gpu.global.or.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.relaxed.sys.global.or.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.acquire.cta.global.or.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.acquire.cluster.global.or.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.acquire.gpu.global.or.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.acquire.sys.global.or.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    }
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.global.op.b64 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .weak }
// .op        = { .xor }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_xor_op_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 multimem_ld_reduce(
  cuda_ptx::sem_weak_t,
  cuda_ptx::op_xor_op_t,
  const B64* addr)
{
  // sem == sem_weak (due to parameter type constraint)
  // op == op_xor_op (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t dest;
    asm (
      "multimem.ld_reduce.weak.global.xor.b64 %0, [%1];"
      : "=l"(dest)
      : "l"(__as_ptr_gmem(addr))
      : "memory"
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.ld_reduce.sem.scope.global.op.b64 dest, [addr]; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .acquire }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .xor }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline B64 multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_xor_op_t,
  const B64* addr);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline B64 multimem_ld_reduce(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_xor_op_t,
  const B64* addr)
{
  static_assert(sem == sem_relaxed || sem == sem_acquire, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_xor_op (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    std::uint64_t dest;
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.relaxed.cta.global.xor.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.relaxed.cluster.global.xor.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.relaxed.gpu.global.xor.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.relaxed.sys.global.xor.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cta) {
      asm (
        "multimem.ld_reduce.acquire.cta.global.xor.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_cluster) {
      asm (
        "multimem.ld_reduce.acquire.cluster.global.xor.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_gpu) {
      asm (
        "multimem.ld_reduce.acquire.gpu.global.xor.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    } else if constexpr (sem == sem_acquire && scope == scope_sys) {
      asm (
        "multimem.ld_reduce.acquire.sys.global.xor.b64 %0, [%1];"
        : "=l"(dest)
        : "l"(__as_ptr_gmem(addr))
        : "memory"
      );
    }
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_ld_reduce_is_not_supported_before_SM_90__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 810
/*
// multimem.st.sem.global.b32 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .weak }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void multimem_st(
  cuda_ptx::sem_weak_t,
  B32* addr,
  B32 val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_st_is_not_supported_before_SM_90__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void multimem_st(
  cuda_ptx::sem_weak_t,
  B32* addr,
  B32 val)
{
  // sem == sem_weak (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "multimem.st.weak.global.b32 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_st_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.st.sem.scope.global.b32 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_st(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  B32* addr,
  B32 val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_st_is_not_supported_before_SM_90__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_st(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  B32* addr,
  B32 val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.st.relaxed.cta.global.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.st.relaxed.cluster.global.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.st.relaxed.gpu.global.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.st.relaxed.sys.global.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.st.release.cta.global.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.st.release.cluster.global.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.st.release.gpu.global.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.st.release.sys.global.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_st_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.st.sem.global.b64 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .weak }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void multimem_st(
  cuda_ptx::sem_weak_t,
  B64* addr,
  B64 val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_st_is_not_supported_before_SM_90__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void multimem_st(
  cuda_ptx::sem_weak_t,
  B64* addr,
  B64 val)
{
  // sem == sem_weak (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "multimem.st.weak.global.b64 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_st_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.st.sem.scope.global.b64 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_st(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  B64* addr,
  B64 val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_st_is_not_supported_before_SM_90__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_st(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  B64* addr,
  B64 val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.st.relaxed.cta.global.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.st.relaxed.cluster.global.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.st.relaxed.gpu.global.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.st.relaxed.sys.global.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.st.release.cta.global.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.st.release.cluster.global.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.st.release.gpu.global.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.st.release.sys.global.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_st_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810
/*
// multimem.red.sem.scope.global.op.u32 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .min }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_min_t,
  std::uint32_t* addr,
  std::uint32_t val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_min_t,
  std::uint32_t* addr,
  std::uint32_t val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.red.relaxed.cta.global.min.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.red.relaxed.cluster.global.min.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.red.relaxed.gpu.global.min.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.red.relaxed.sys.global.min.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.red.release.cta.global.min.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.red.release.cluster.global.min.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.red.release.gpu.global.min.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.red.release.sys.global.min.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.red.sem.scope.global.op.u64 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .min }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_min_t,
  std::uint64_t* addr,
  std::uint64_t val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_min_t,
  std::uint64_t* addr,
  std::uint64_t val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.red.relaxed.cta.global.min.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.red.relaxed.cluster.global.min.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.red.relaxed.gpu.global.min.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.red.relaxed.sys.global.min.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.red.release.cta.global.min.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.red.release.cluster.global.min.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.red.release.gpu.global.min.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.red.release.sys.global.min.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.red.sem.scope.global.op.s32 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .min }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_min_t,
  std::int32_t* addr,
  std::int32_t val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_min_t,
  std::int32_t* addr,
  std::int32_t val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.red.relaxed.cta.global.min.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.red.relaxed.cluster.global.min.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.red.relaxed.gpu.global.min.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.red.relaxed.sys.global.min.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.red.release.cta.global.min.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.red.release.cluster.global.min.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.red.release.gpu.global.min.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.red.release.sys.global.min.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.red.sem.scope.global.op.s64 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .min }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_min_t,
  std::int64_t* addr,
  std::int64_t val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_min_t,
  std::int64_t* addr,
  std::int64_t val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.red.relaxed.cta.global.min.s64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.red.relaxed.cluster.global.min.s64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.red.relaxed.gpu.global.min.s64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.red.relaxed.sys.global.min.s64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.red.release.cta.global.min.s64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.red.release.cluster.global.min.s64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.red.release.gpu.global.min.s64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.red.release.sys.global.min.s64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.red.sem.scope.global.op.u32 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .max }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_max_t,
  std::uint32_t* addr,
  std::uint32_t val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_max_t,
  std::uint32_t* addr,
  std::uint32_t val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.red.relaxed.cta.global.max.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.red.relaxed.cluster.global.max.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.red.relaxed.gpu.global.max.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.red.relaxed.sys.global.max.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.red.release.cta.global.max.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.red.release.cluster.global.max.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.red.release.gpu.global.max.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.red.release.sys.global.max.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.red.sem.scope.global.op.u64 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .max }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_max_t,
  std::uint64_t* addr,
  std::uint64_t val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_max_t,
  std::uint64_t* addr,
  std::uint64_t val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.red.relaxed.cta.global.max.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.red.relaxed.cluster.global.max.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.red.relaxed.gpu.global.max.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.red.relaxed.sys.global.max.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.red.release.cta.global.max.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.red.release.cluster.global.max.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.red.release.gpu.global.max.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.red.release.sys.global.max.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.red.sem.scope.global.op.s32 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .max }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_max_t,
  std::int32_t* addr,
  std::int32_t val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_max_t,
  std::int32_t* addr,
  std::int32_t val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.red.relaxed.cta.global.max.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.red.relaxed.cluster.global.max.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.red.relaxed.gpu.global.max.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.red.relaxed.sys.global.max.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.red.release.cta.global.max.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.red.release.cluster.global.max.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.red.release.gpu.global.max.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.red.release.sys.global.max.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.red.sem.scope.global.op.s64 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .max }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_max_t,
  std::int64_t* addr,
  std::int64_t val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_max_t,
  std::int64_t* addr,
  std::int64_t val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.red.relaxed.cta.global.max.s64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.red.relaxed.cluster.global.max.s64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.red.relaxed.gpu.global.max.s64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.red.relaxed.sys.global.max.s64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.red.release.cta.global.max.s64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.red.release.cluster.global.max.s64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.red.release.gpu.global.max.s64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.red.release.sys.global.max.s64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.red.sem.scope.global.op.u32 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .add }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_add_t,
  std::uint32_t* addr,
  std::uint32_t val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_add_t,
  std::uint32_t* addr,
  std::uint32_t val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.red.relaxed.cta.global.add.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.red.relaxed.cluster.global.add.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.red.relaxed.gpu.global.add.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.red.relaxed.sys.global.add.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.red.release.cta.global.add.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.red.release.cluster.global.add.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.red.release.gpu.global.add.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.red.release.sys.global.add.u32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.red.sem.scope.global.op.u64 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .add }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_add_t,
  std::uint64_t* addr,
  std::uint64_t val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_add_t,
  std::uint64_t* addr,
  std::uint64_t val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.red.relaxed.cta.global.add.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.red.relaxed.cluster.global.add.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.red.relaxed.gpu.global.add.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.red.relaxed.sys.global.add.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.red.release.cta.global.add.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.red.release.cluster.global.add.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.red.release.gpu.global.add.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.red.release.sys.global.add.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.red.sem.scope.global.op.s32 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .add }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_add_t,
  std::int32_t* addr,
  std::int32_t val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_add_t,
  std::int32_t* addr,
  std::int32_t val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.red.relaxed.cta.global.add.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.red.relaxed.cluster.global.add.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.red.relaxed.gpu.global.add.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.red.relaxed.sys.global.add.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.red.release.cta.global.add.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.red.release.cluster.global.add.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.red.release.gpu.global.add.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.red.release.sys.global.add.s32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(val)
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.red.sem.scope.global.op.u64 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .add }
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_add_t,
  std::int64_t* addr,
  std::int64_t val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
template <cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_add_t,
  std::int64_t* addr,
  std::int64_t val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.red.relaxed.cta.global.add.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.red.relaxed.cluster.global.add.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.red.relaxed.gpu.global.add.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.red.relaxed.sys.global.add.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.red.release.cta.global.add.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.red.release.cluster.global.add.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.red.release.gpu.global.add.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.red.release.sys.global.add.u64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(val)
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.red.sem.scope.global.op.b32 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .and }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_and_op_t,
  B32* addr,
  B32 val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_and_op_t,
  B32* addr,
  B32 val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_and_op (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.red.relaxed.cta.global.and.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.red.relaxed.cluster.global.and.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.red.relaxed.gpu.global.and.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.red.relaxed.sys.global.and.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.red.release.cta.global.and.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.red.release.cluster.global.and.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.red.release.gpu.global.and.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.red.release.sys.global.and.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.red.sem.scope.global.op.b32 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .or }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_or_op_t,
  B32* addr,
  B32 val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_or_op_t,
  B32* addr,
  B32 val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_or_op (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.red.relaxed.cta.global.or.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.red.relaxed.cluster.global.or.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.red.relaxed.gpu.global.or.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.red.relaxed.sys.global.or.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.red.release.cta.global.or.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.red.release.cluster.global.or.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.red.release.gpu.global.or.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.red.release.sys.global.or.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.red.sem.scope.global.op.b32 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .xor }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_xor_op_t,
  B32* addr,
  B32 val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_xor_op_t,
  B32* addr,
  B32 val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_xor_op (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.red.relaxed.cta.global.xor.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.red.relaxed.cluster.global.xor.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.red.relaxed.gpu.global.xor.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.red.relaxed.sys.global.xor.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.red.release.cta.global.xor.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.red.release.cluster.global.xor.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.red.release.gpu.global.xor.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.red.release.sys.global.xor.b32 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&val))
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.red.sem.scope.global.op.b64 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .and }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_and_op_t,
  B64* addr,
  B64 val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_and_op_t,
  B64* addr,
  B64 val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_and_op (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.red.relaxed.cta.global.and.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.red.relaxed.cluster.global.and.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.red.relaxed.gpu.global.and.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.red.relaxed.sys.global.and.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.red.release.cta.global.and.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.red.release.cluster.global.and.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.red.release.gpu.global.and.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.red.release.sys.global.and.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.red.sem.scope.global.op.b64 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .or }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_or_op_t,
  B64* addr,
  B64 val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_or_op_t,
  B64* addr,
  B64 val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_or_op (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.red.relaxed.cta.global.or.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.red.relaxed.cluster.global.or.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.red.relaxed.gpu.global.or.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.red.relaxed.sys.global.or.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.red.release.cta.global.or.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.red.release.cluster.global.or.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.red.release.gpu.global.or.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.red.release.sys.global.or.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// multimem.red.sem.scope.global.op.b64 [addr], val; // PTX ISA 81, SM_90
// .sem       = { .relaxed, .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
// .op        = { .xor }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_xor_op_t,
  B64* addr,
  B64 val);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true, cuda_ptx::dot_sem Sem, cuda_ptx::dot_scope Scope>
__device__ static inline void multimem_red(
  cuda_ptx::sem_t<Sem> sem,
  cuda_ptx::scope_t<Scope> scope,
  cuda_ptx::op_xor_op_t,
  B64* addr,
  B64 val)
{
  static_assert(sem == sem_relaxed || sem == sem_release, "");
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  // op == op_xor_op (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sem == sem_relaxed && scope == scope_cta) {
      asm (
        "multimem.red.relaxed.cta.global.xor.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_cluster) {
      asm (
        "multimem.red.relaxed.cluster.global.xor.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_gpu) {
      asm (
        "multimem.red.relaxed.gpu.global.xor.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_relaxed && scope == scope_sys) {
      asm (
        "multimem.red.relaxed.sys.global.xor.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cta) {
      asm (
        "multimem.red.release.cta.global.xor.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_cluster) {
      asm (
        "multimem.red.release.cluster.global.xor.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_gpu) {
      asm (
        "multimem.red.release.gpu.global.xor.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    } else if constexpr (sem == sem_release && scope == scope_sys) {
      asm (
        "multimem.red.release.sys.global.xor.b64 [%0], %1;"
        :
        : "l"(__as_ptr_gmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&val))
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_multimem_red_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

} // namespace cuda_ptx

#endif // INSTRUCTIONS_MULTIMEM_H

// End of ./instructions/multimem.h
// Start of ./instructions/prmt.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_PRMT_H
#define INSTRUCTIONS_PRMT_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/prmt.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// prmt.b32 dest, a_reg, b_reg, c_reg; // PTX ISA 20, SM_50
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline std::uint32_t prmt(
  B32 a_reg,
  B32 b_reg,
  std::uint32_t c_reg);
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_prmt_is_not_supported_before_SM_50__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline std::uint32_t prmt(
  B32 a_reg,
  B32 b_reg,
  std::uint32_t c_reg)
{
  static_assert(sizeof(B32) == 4, "");
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "prmt.b32 %0, %1, %2, %3;"
      : "=r"(dest)
      : "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&a_reg)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&b_reg)),
        "r"(c_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_prmt_is_not_supported_before_SM_50__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// prmt.b32.f4e dest, a_reg, b_reg, c_reg; // PTX ISA 20, SM_50
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline std::uint32_t prmt_f4e(
  B32 a_reg,
  B32 b_reg,
  std::uint32_t c_reg);
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_prmt_f4e_is_not_supported_before_SM_50__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline std::uint32_t prmt_f4e(
  B32 a_reg,
  B32 b_reg,
  std::uint32_t c_reg)
{
  static_assert(sizeof(B32) == 4, "");
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "prmt.b32.f4e %0, %1, %2, %3;"
      : "=r"(dest)
      : "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&a_reg)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&b_reg)),
        "r"(c_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_prmt_f4e_is_not_supported_before_SM_50__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// prmt.b32.b4e dest, a_reg, b_reg, c_reg; // PTX ISA 20, SM_50
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline std::uint32_t prmt_b4e(
  B32 a_reg,
  B32 b_reg,
  std::uint32_t c_reg);
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_prmt_b4e_is_not_supported_before_SM_50__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline std::uint32_t prmt_b4e(
  B32 a_reg,
  B32 b_reg,
  std::uint32_t c_reg)
{
  static_assert(sizeof(B32) == 4, "");
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "prmt.b32.b4e %0, %1, %2, %3;"
      : "=r"(dest)
      : "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&a_reg)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&b_reg)),
        "r"(c_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_prmt_b4e_is_not_supported_before_SM_50__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// prmt.b32.rc8 dest, a_reg, b_reg, c_reg; // PTX ISA 20, SM_50
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline std::uint32_t prmt_rc8(
  B32 a_reg,
  B32 b_reg,
  std::uint32_t c_reg);
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_prmt_rc8_is_not_supported_before_SM_50__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline std::uint32_t prmt_rc8(
  B32 a_reg,
  B32 b_reg,
  std::uint32_t c_reg)
{
  static_assert(sizeof(B32) == 4, "");
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "prmt.b32.rc8 %0, %1, %2, %3;"
      : "=r"(dest)
      : "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&a_reg)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&b_reg)),
        "r"(c_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_prmt_rc8_is_not_supported_before_SM_50__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// prmt.b32.ecl dest, a_reg, b_reg, c_reg; // PTX ISA 20, SM_50
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline std::uint32_t prmt_ecl(
  B32 a_reg,
  B32 b_reg,
  std::uint32_t c_reg);
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_prmt_ecl_is_not_supported_before_SM_50__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline std::uint32_t prmt_ecl(
  B32 a_reg,
  B32 b_reg,
  std::uint32_t c_reg)
{
  static_assert(sizeof(B32) == 4, "");
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "prmt.b32.ecl %0, %1, %2, %3;"
      : "=r"(dest)
      : "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&a_reg)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&b_reg)),
        "r"(c_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_prmt_ecl_is_not_supported_before_SM_50__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// prmt.b32.ecr dest, a_reg, b_reg, c_reg; // PTX ISA 20, SM_50
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline std::uint32_t prmt_ecr(
  B32 a_reg,
  B32 b_reg,
  std::uint32_t c_reg);
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_prmt_ecr_is_not_supported_before_SM_50__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline std::uint32_t prmt_ecr(
  B32 a_reg,
  B32 b_reg,
  std::uint32_t c_reg)
{
  static_assert(sizeof(B32) == 4, "");
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "prmt.b32.ecr %0, %1, %2, %3;"
      : "=r"(dest)
      : "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&a_reg)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&b_reg)),
        "r"(c_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_prmt_ecr_is_not_supported_before_SM_50__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

/*
// prmt.b32.rc16 dest, a_reg, b_reg, c_reg; // PTX ISA 20, SM_50
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline std::uint32_t prmt_rc16(
  B32 a_reg,
  B32 b_reg,
  std::uint32_t c_reg);
*/
#if __libcuda_ptx_isa >= 200
extern "C" __device__ void __cuda_ptx_prmt_rc16_is_not_supported_before_SM_50__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline std::uint32_t prmt_rc16(
  B32 a_reg,
  B32 b_reg,
  std::uint32_t c_reg)
{
  static_assert(sizeof(B32) == 4, "");
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "prmt.b32.rc16 %0, %1, %2, %3;"
      : "=r"(dest)
      : "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&a_reg)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&b_reg)),
        "r"(c_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_prmt_rc16_is_not_supported_before_SM_50__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 200

} // namespace cuda_ptx

#endif // INSTRUCTIONS_PRMT_H

// End of ./instructions/prmt.h
// Start of ./instructions/red_async.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_RED_ASYNC_H
#define INSTRUCTIONS_RED_ASYNC_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/red_async.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type  [dest], value, [remote_bar];  // PTX ISA 81, SM_90
// .type      = { .u32 }
// .op        = { .inc }
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_inc_t,
  std::uint32_t* dest,
  const std::uint32_t& value,
  std::uint64_t* remote_bar);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_inc_t,
  std::uint32_t* dest,
  const std::uint32_t& value,
  std::uint64_t* remote_bar)
{
  // type == type_u32 (due to parameter type constraint)
  // op == op_inc (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.inc.u32  [%0], %1, [%2]; "
      :
      : "r"(__as_ptr_remote_dsmem(dest)),
        "r"(value),
        "r"(__as_ptr_remote_dsmem(remote_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_red_async_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type  [dest], value, [remote_bar];  // PTX ISA 81, SM_90
// .type      = { .u32 }
// .op        = { .dec }
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_dec_t,
  std::uint32_t* dest,
  const std::uint32_t& value,
  std::uint64_t* remote_bar);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_dec_t,
  std::uint32_t* dest,
  const std::uint32_t& value,
  std::uint64_t* remote_bar)
{
  // type == type_u32 (due to parameter type constraint)
  // op == op_dec (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.dec.u32  [%0], %1, [%2]; "
      :
      : "r"(__as_ptr_remote_dsmem(dest)),
        "r"(value),
        "r"(__as_ptr_remote_dsmem(remote_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_red_async_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type  [dest], value, [remote_bar];  // PTX ISA 81, SM_90
// .type      = { .u32 }
// .op        = { .min }
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_min_t,
  std::uint32_t* dest,
  const std::uint32_t& value,
  std::uint64_t* remote_bar);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_min_t,
  std::uint32_t* dest,
  const std::uint32_t& value,
  std::uint64_t* remote_bar)
{
  // type == type_u32 (due to parameter type constraint)
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.min.u32  [%0], %1, [%2]; "
      :
      : "r"(__as_ptr_remote_dsmem(dest)),
        "r"(value),
        "r"(__as_ptr_remote_dsmem(remote_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_red_async_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type  [dest], value, [remote_bar];  // PTX ISA 81, SM_90
// .type      = { .u32 }
// .op        = { .max }
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_max_t,
  std::uint32_t* dest,
  const std::uint32_t& value,
  std::uint64_t* remote_bar);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_max_t,
  std::uint32_t* dest,
  const std::uint32_t& value,
  std::uint64_t* remote_bar)
{
  // type == type_u32 (due to parameter type constraint)
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.max.u32  [%0], %1, [%2]; "
      :
      : "r"(__as_ptr_remote_dsmem(dest)),
        "r"(value),
        "r"(__as_ptr_remote_dsmem(remote_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_red_async_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type  [dest], value, [remote_bar];  // PTX ISA 81, SM_90
// .type      = { .u32 }
// .op        = { .add }
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_add_t,
  std::uint32_t* dest,
  const std::uint32_t& value,
  std::uint64_t* remote_bar);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_add_t,
  std::uint32_t* dest,
  const std::uint32_t& value,
  std::uint64_t* remote_bar)
{
  // type == type_u32 (due to parameter type constraint)
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.add.u32  [%0], %1, [%2]; "
      :
      : "r"(__as_ptr_remote_dsmem(dest)),
        "r"(value),
        "r"(__as_ptr_remote_dsmem(remote_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_red_async_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type  [dest], value, [remote_bar];  // PTX ISA 81, SM_90
// .type      = { .s32 }
// .op        = { .min }
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_min_t,
  std::int32_t* dest,
  const std::int32_t& value,
  std::uint64_t* remote_bar);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_min_t,
  std::int32_t* dest,
  const std::int32_t& value,
  std::uint64_t* remote_bar)
{
  // type == type_s32 (due to parameter type constraint)
  // op == op_min (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.min.s32  [%0], %1, [%2]; "
      :
      : "r"(__as_ptr_remote_dsmem(dest)),
        "r"(value),
        "r"(__as_ptr_remote_dsmem(remote_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_red_async_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type  [dest], value, [remote_bar];  // PTX ISA 81, SM_90
// .type      = { .s32 }
// .op        = { .max }
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_max_t,
  std::int32_t* dest,
  const std::int32_t& value,
  std::uint64_t* remote_bar);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_max_t,
  std::int32_t* dest,
  const std::int32_t& value,
  std::uint64_t* remote_bar)
{
  // type == type_s32 (due to parameter type constraint)
  // op == op_max (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.max.s32  [%0], %1, [%2]; "
      :
      : "r"(__as_ptr_remote_dsmem(dest)),
        "r"(value),
        "r"(__as_ptr_remote_dsmem(remote_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_red_async_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type  [dest], value, [remote_bar];  // PTX ISA 81, SM_90
// .type      = { .s32 }
// .op        = { .add }
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_add_t,
  std::int32_t* dest,
  const std::int32_t& value,
  std::uint64_t* remote_bar);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_add_t,
  std::int32_t* dest,
  const std::int32_t& value,
  std::uint64_t* remote_bar)
{
  // type == type_s32 (due to parameter type constraint)
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.add.s32  [%0], %1, [%2]; "
      :
      : "r"(__as_ptr_remote_dsmem(dest)),
        "r"(value),
        "r"(__as_ptr_remote_dsmem(remote_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_red_async_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type  [dest], value, [remote_bar];  // PTX ISA 81, SM_90
// .type      = { .b32 }
// .op        = { .and }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void red_async(
  cuda_ptx::op_and_op_t,
  B32* dest,
  const B32& value,
  std::uint64_t* remote_bar);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void red_async(
  cuda_ptx::op_and_op_t,
  B32* dest,
  const B32& value,
  std::uint64_t* remote_bar)
{
  // type == type_b32 (due to parameter type constraint)
  // op == op_and_op (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.and.b32  [%0], %1, [%2]; "
      :
      : "r"(__as_ptr_remote_dsmem(dest)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&value)),
        "r"(__as_ptr_remote_dsmem(remote_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_red_async_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type  [dest], value, [remote_bar];  // PTX ISA 81, SM_90
// .type      = { .b32 }
// .op        = { .or }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void red_async(
  cuda_ptx::op_or_op_t,
  B32* dest,
  const B32& value,
  std::uint64_t* remote_bar);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void red_async(
  cuda_ptx::op_or_op_t,
  B32* dest,
  const B32& value,
  std::uint64_t* remote_bar)
{
  // type == type_b32 (due to parameter type constraint)
  // op == op_or_op (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.or.b32  [%0], %1, [%2]; "
      :
      : "r"(__as_ptr_remote_dsmem(dest)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&value)),
        "r"(__as_ptr_remote_dsmem(remote_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_red_async_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type  [dest], value, [remote_bar];  // PTX ISA 81, SM_90
// .type      = { .b32 }
// .op        = { .xor }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void red_async(
  cuda_ptx::op_xor_op_t,
  B32* dest,
  const B32& value,
  std::uint64_t* remote_bar);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void red_async(
  cuda_ptx::op_xor_op_t,
  B32* dest,
  const B32& value,
  std::uint64_t* remote_bar)
{
  // type == type_b32 (due to parameter type constraint)
  // op == op_xor_op (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.xor.b32  [%0], %1, [%2]; "
      :
      : "r"(__as_ptr_remote_dsmem(dest)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&value)),
        "r"(__as_ptr_remote_dsmem(remote_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_red_async_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.type  [dest], value, [remote_bar];  // PTX ISA 81, SM_90
// .type      = { .u64 }
// .op        = { .add }
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_add_t,
  std::uint64_t* dest,
  const std::uint64_t& value,
  std::uint64_t* remote_bar);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_add_t,
  std::uint64_t* dest,
  const std::uint64_t& value,
  std::uint64_t* remote_bar)
{
  // type == type_u64 (due to parameter type constraint)
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.add.u64  [%0], %1, [%2]; "
      :
      : "r"(__as_ptr_remote_dsmem(dest)),
        "l"(value),
        "r"(__as_ptr_remote_dsmem(remote_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_red_async_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.op.u64  [dest], value, [remote_bar]; // .u64 intentional PTX ISA 81, SM_90
// .op        = { .add }
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_add_t,
  std::int64_t* dest,
  const std::int64_t& value,
  std::int64_t* remote_bar);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_red_async_is_not_supported_before_SM_90__();
template <typename = void>
__device__ static inline void red_async(
  cuda_ptx::op_add_t,
  std::int64_t* dest,
  const std::int64_t& value,
  std::int64_t* remote_bar)
{
  // op == op_add (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "red.async.relaxed.cluster.shared::cluster.mbarrier::complete_tx::bytes.add.u64  [%0], %1, [%2]; // .u64 intentional"
      :
      : "r"(__as_ptr_remote_dsmem(dest)),
        "l"(value),
        "r"(__as_ptr_remote_dsmem(remote_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_red_async_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

} // namespace cuda_ptx

#endif // INSTRUCTIONS_RED_ASYNC_H

// End of ./instructions/red_async.h
// Start of ./instructions/setmaxnreg.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_SETMAXNREG_H
#define INSTRUCTIONS_SETMAXNREG_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/setmaxnreg.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// setmaxnreg.inc.sync.aligned.u32 imm_reg_count; // PTX ISA 80, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
template <int N32>
__device__ static inline void setmaxnreg_inc(
  cuda_ptx::n32_t<N32> imm_reg_count);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_setmaxnreg_inc_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32>
__device__ static inline void setmaxnreg_inc(
  cuda_ptx::n32_t<N32> imm_reg_count)
{
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm volatile (
      "setmaxnreg.inc.sync.aligned.u32 %0;"
      :
      : "n"(imm_reg_count.value)
      :
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_setmaxnreg_inc_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

/*
// setmaxnreg.dec.sync.aligned.u32 imm_reg_count; // PTX ISA 80, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
template <int N32>
__device__ static inline void setmaxnreg_dec(
  cuda_ptx::n32_t<N32> imm_reg_count);
*/
#if __libcuda_ptx_isa >= 800
extern "C" __device__ void __cuda_ptx_setmaxnreg_dec_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32>
__device__ static inline void setmaxnreg_dec(
  cuda_ptx::n32_t<N32> imm_reg_count)
{
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm volatile (
      "setmaxnreg.dec.sync.aligned.u32 %0;"
      :
      : "n"(imm_reg_count.value)
      :
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_setmaxnreg_dec_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 800

} // namespace cuda_ptx

#endif // INSTRUCTIONS_SETMAXNREG_H

// End of ./instructions/setmaxnreg.h
// Start of ./instructions/shl.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_SHL_H
#define INSTRUCTIONS_SHL_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/shl.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// shl.b16 dest, a_reg, b_reg; // PTX ISA 10, SM_50
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 shl(
  B16 a_reg,
  std::uint32_t b_reg);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_shl_is_not_supported_before_SM_50__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 shl(
  B16 a_reg,
  std::uint32_t b_reg)
{
  static_assert(sizeof(B16) == 2, "");
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint16_t dest;
    asm (
      "shl.b16 %0, %1, %2;"
      : "=h"(dest)
      : "h"(/*as_b16*/ *reinterpret_cast<const std::int16_t*>(&a_reg)),
        "r"(b_reg)
      :
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_shl_is_not_supported_before_SM_50__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 100

/*
// shl.b32 dest, a_reg, b_reg; // PTX ISA 10, SM_50
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 shl(
  B32 a_reg,
  std::uint32_t b_reg);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_shl_is_not_supported_before_SM_50__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 shl(
  B32 a_reg,
  std::uint32_t b_reg)
{
  static_assert(sizeof(B32) == 4, "");
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "shl.b32 %0, %1, %2;"
      : "=r"(dest)
      : "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&a_reg)),
        "r"(b_reg)
      :
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_shl_is_not_supported_before_SM_50__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 100

/*
// shl.b64 dest, a_reg, b_reg; // PTX ISA 10, SM_50
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 shl(
  B64 a_reg,
  std::uint32_t b_reg);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_shl_is_not_supported_before_SM_50__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 shl(
  B64 a_reg,
  std::uint32_t b_reg)
{
  static_assert(sizeof(B64) == 8, "");
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint64_t dest;
    asm (
      "shl.b64 %0, %1, %2;"
      : "=l"(dest)
      : "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&a_reg)),
        "r"(b_reg)
      :
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_shl_is_not_supported_before_SM_50__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 100

} // namespace cuda_ptx

#endif // INSTRUCTIONS_SHL_H

// End of ./instructions/shl.h
// Start of ./instructions/shr.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_SHR_H
#define INSTRUCTIONS_SHR_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/shr.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// shr.b16 dest, a_reg, b_reg; // PTX ISA 10, SM_50
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 shr(
  B16 a_reg,
  std::uint32_t b_reg);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_shr_is_not_supported_before_SM_50__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline B16 shr(
  B16 a_reg,
  std::uint32_t b_reg)
{
  static_assert(sizeof(B16) == 2, "");
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint16_t dest;
    asm (
      "shr.b16 %0, %1, %2;"
      : "=h"(dest)
      : "h"(/*as_b16*/ *reinterpret_cast<const std::int16_t*>(&a_reg)),
        "r"(b_reg)
      :
    );
    return *reinterpret_cast<B16*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_shr_is_not_supported_before_SM_50__();
    std::uint16_t err_out_var = 0;
    return *reinterpret_cast<B16*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 100

/*
// shr.b32 dest, a_reg, b_reg; // PTX ISA 10, SM_50
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 shr(
  B32 a_reg,
  std::uint32_t b_reg);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_shr_is_not_supported_before_SM_50__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline B32 shr(
  B32 a_reg,
  std::uint32_t b_reg)
{
  static_assert(sizeof(B32) == 4, "");
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint32_t dest;
    asm (
      "shr.b32 %0, %1, %2;"
      : "=r"(dest)
      : "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&a_reg)),
        "r"(b_reg)
      :
    );
    return *reinterpret_cast<B32*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_shr_is_not_supported_before_SM_50__();
    std::uint32_t err_out_var = 0;
    return *reinterpret_cast<B32*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 100

/*
// shr.b64 dest, a_reg, b_reg; // PTX ISA 10, SM_50
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 shr(
  B64 a_reg,
  std::uint32_t b_reg);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_shr_is_not_supported_before_SM_50__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline B64 shr(
  B64 a_reg,
  std::uint32_t b_reg)
{
  static_assert(sizeof(B64) == 8, "");
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::uint64_t dest;
    asm (
      "shr.b64 %0, %1, %2;"
      : "=l"(dest)
      : "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&a_reg)),
        "r"(b_reg)
      :
    );
    return *reinterpret_cast<B64*>(&dest);
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_shr_is_not_supported_before_SM_50__();
    std::uint64_t err_out_var = 0;
    return *reinterpret_cast<B64*>(&err_out_var);
  #endif
}
#endif // __libcuda_ptx_isa >= 100

/*
// shr.s16 dest, a_reg, b_reg; // PTX ISA 10, SM_50
template <typename = void>
__device__ static inline std::int16_t shr(
  std::int16_t a_reg,
  std::uint32_t b_reg);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_shr_is_not_supported_before_SM_50__();
template <typename = void>
__device__ static inline std::int16_t shr(
  std::int16_t a_reg,
  std::uint32_t b_reg)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::int16_t dest;
    asm (
      "shr.s16 %0, %1, %2;"
      : "=h"(dest)
      : "h"(a_reg),
        "r"(b_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_shr_is_not_supported_before_SM_50__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 100

/*
// shr.s32 dest, a_reg, b_reg; // PTX ISA 10, SM_50
template <typename = void>
__device__ static inline std::int32_t shr(
  std::int32_t a_reg,
  std::uint32_t b_reg);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_shr_is_not_supported_before_SM_50__();
template <typename = void>
__device__ static inline std::int32_t shr(
  std::int32_t a_reg,
  std::uint32_t b_reg)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::int32_t dest;
    asm (
      "shr.s32 %0, %1, %2;"
      : "=r"(dest)
      : "r"(a_reg),
        "r"(b_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_shr_is_not_supported_before_SM_50__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 100

/*
// shr.s64 dest, a_reg, b_reg; // PTX ISA 10, SM_50
template <typename = void>
__device__ static inline std::int64_t shr(
  std::int64_t a_reg,
  std::uint32_t b_reg);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_shr_is_not_supported_before_SM_50__();
template <typename = void>
__device__ static inline std::int64_t shr(
  std::int64_t a_reg,
  std::uint32_t b_reg)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    std::int64_t dest;
    asm (
      "shr.s64 %0, %1, %2;"
      : "=l"(dest)
      : "l"(a_reg),
        "r"(b_reg)
      :
    );
    return dest;
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_shr_is_not_supported_before_SM_50__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 100

} // namespace cuda_ptx

#endif // INSTRUCTIONS_SHR_H

// End of ./instructions/shr.h
// Start of ./instructions/st.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_ST_H
#define INSTRUCTIONS_ST_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/st.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// st.space.b8 [addr], src; // PTX ISA 10, SM_50
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void st(
  cuda_ptx::space_global_t,
  B8* addr,
  B8 src);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_st_is_not_supported_before_SM_50__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void st(
  cuda_ptx::space_global_t,
  B8* addr,
  B8 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    asm (
      "st.global.b8 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "r"(__b8_as_u32(src))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_is_not_supported_before_SM_50__();
  #endif
}
#endif // __libcuda_ptx_isa >= 100

/*
// st.space.b16 [addr], src; // PTX ISA 10, SM_50
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void st(
  cuda_ptx::space_global_t,
  B16* addr,
  B16 src);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_st_is_not_supported_before_SM_50__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void st(
  cuda_ptx::space_global_t,
  B16* addr,
  B16 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    asm (
      "st.global.b16 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "h"(/*as_b16*/ *reinterpret_cast<const std::int16_t*>(&src))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_is_not_supported_before_SM_50__();
  #endif
}
#endif // __libcuda_ptx_isa >= 100

/*
// st.space.b32 [addr], src; // PTX ISA 10, SM_50
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void st(
  cuda_ptx::space_global_t,
  B32* addr,
  B32 src);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_st_is_not_supported_before_SM_50__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void st(
  cuda_ptx::space_global_t,
  B32* addr,
  B32 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    asm (
      "st.global.b32 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&src))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_is_not_supported_before_SM_50__();
  #endif
}
#endif // __libcuda_ptx_isa >= 100

/*
// st.space.b64 [addr], src; // PTX ISA 10, SM_50
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void st(
  cuda_ptx::space_global_t,
  B64* addr,
  B64 src);
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_st_is_not_supported_before_SM_50__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void st(
  cuda_ptx::space_global_t,
  B64* addr,
  B64 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    asm (
      "st.global.b64 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&src))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_is_not_supported_before_SM_50__();
  #endif
}
#endif // __libcuda_ptx_isa >= 100

/*
// st.space.b128 [addr], src; // PTX ISA 83, SM_70
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline void st(
  cuda_ptx::space_global_t,
  B128* addr,
  B128 src);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_st_is_not_supported_before_SM_70__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline void st(
  cuda_ptx::space_global_t,
  B128* addr,
  B128 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    asm (
      "{\n\t"
      ".reg .b128 B128_src; \n\t"
      "mov.b128 B128_src, {%1, %2}; \n\t"
      "st.global.b128 [%0], B128_src; \n"
      "}"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"((*reinterpret_cast<longlong2*>(&src)).x),
        "l"((*reinterpret_cast<longlong2*>(&src)).y)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_is_not_supported_before_SM_70__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// st.space.v4.b64 [addr], src; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline void st(
  cuda_ptx::space_global_t,
  B256* addr,
  B256 src);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_st_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline void st(
  cuda_ptx::space_global_t,
  B256* addr,
  B256 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    asm (
      "st.global.v4.b64 [%0], {%1, %2, %3, %4};"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).x),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).y),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).z),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).w)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_is_not_supported_before_SM_100__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// st.space.L2::cache_hint.b8 [addr], src, cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void st_L2_cache_hint(
  cuda_ptx::space_global_t,
  B8* addr,
  B8 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void st_L2_cache_hint(
  cuda_ptx::space_global_t,
  B8* addr,
  B8 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "st.global.L2::cache_hint.b8 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "r"(__b8_as_u32(src)),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L2::cache_hint.b16 [addr], src, cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void st_L2_cache_hint(
  cuda_ptx::space_global_t,
  B16* addr,
  B16 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void st_L2_cache_hint(
  cuda_ptx::space_global_t,
  B16* addr,
  B16 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "st.global.L2::cache_hint.b16 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "h"(/*as_b16*/ *reinterpret_cast<const std::int16_t*>(&src)),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L2::cache_hint.b32 [addr], src, cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void st_L2_cache_hint(
  cuda_ptx::space_global_t,
  B32* addr,
  B32 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void st_L2_cache_hint(
  cuda_ptx::space_global_t,
  B32* addr,
  B32 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "st.global.L2::cache_hint.b32 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&src)),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L2::cache_hint.b64 [addr], src, cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void st_L2_cache_hint(
  cuda_ptx::space_global_t,
  B64* addr,
  B64 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void st_L2_cache_hint(
  cuda_ptx::space_global_t,
  B64* addr,
  B64 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "st.global.L2::cache_hint.b64 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&src)),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L2::cache_hint.b128 [addr], src, cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline void st_L2_cache_hint(
  cuda_ptx::space_global_t,
  B128* addr,
  B128 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_st_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline void st_L2_cache_hint(
  cuda_ptx::space_global_t,
  B128* addr,
  B128 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "{\n\t"
      ".reg .b128 B128_src; \n\t"
      "mov.b128 B128_src, {%1, %2}; \n\t"
      "st.global.L2::cache_hint.b128 [%0], B128_src, %3; \n"
      "}"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"((*reinterpret_cast<longlong2*>(&src)).x),
        "l"((*reinterpret_cast<longlong2*>(&src)).y),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// st.space.L2::cache_hint.v4.b64 [addr], src, cache_policy; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline void st_L2_cache_hint(
  cuda_ptx::space_global_t,
  B256* addr,
  B256 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_st_L2_cache_hint_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline void st_L2_cache_hint(
  cuda_ptx::space_global_t,
  B256* addr,
  B256 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    asm (
      "st.global.L2::cache_hint.v4.b64 [%0], {%1, %2, %3, %4}, %5;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).x),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).y),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).z),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).w),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L2_cache_hint_is_not_supported_before_SM_100__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// st.space.L1::evict_first.b8 [addr], src; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void st_L1_evict_first(
  cuda_ptx::space_global_t,
  B8* addr,
  B8 src);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_evict_first_is_not_supported_before_SM_70__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void st_L1_evict_first(
  cuda_ptx::space_global_t,
  B8* addr,
  B8 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    asm (
      "st.global.L1::evict_first.b8 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "r"(__b8_as_u32(src))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_first_is_not_supported_before_SM_70__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::evict_first.b16 [addr], src; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void st_L1_evict_first(
  cuda_ptx::space_global_t,
  B16* addr,
  B16 src);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_evict_first_is_not_supported_before_SM_70__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void st_L1_evict_first(
  cuda_ptx::space_global_t,
  B16* addr,
  B16 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    asm (
      "st.global.L1::evict_first.b16 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "h"(/*as_b16*/ *reinterpret_cast<const std::int16_t*>(&src))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_first_is_not_supported_before_SM_70__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::evict_first.b32 [addr], src; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void st_L1_evict_first(
  cuda_ptx::space_global_t,
  B32* addr,
  B32 src);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_evict_first_is_not_supported_before_SM_70__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void st_L1_evict_first(
  cuda_ptx::space_global_t,
  B32* addr,
  B32 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    asm (
      "st.global.L1::evict_first.b32 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&src))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_first_is_not_supported_before_SM_70__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::evict_first.b64 [addr], src; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void st_L1_evict_first(
  cuda_ptx::space_global_t,
  B64* addr,
  B64 src);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_evict_first_is_not_supported_before_SM_70__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void st_L1_evict_first(
  cuda_ptx::space_global_t,
  B64* addr,
  B64 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    asm (
      "st.global.L1::evict_first.b64 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&src))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_first_is_not_supported_before_SM_70__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::evict_first.b128 [addr], src; // PTX ISA 83, SM_70
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline void st_L1_evict_first(
  cuda_ptx::space_global_t,
  B128* addr,
  B128 src);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_st_L1_evict_first_is_not_supported_before_SM_70__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline void st_L1_evict_first(
  cuda_ptx::space_global_t,
  B128* addr,
  B128 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    asm (
      "{\n\t"
      ".reg .b128 B128_src; \n\t"
      "mov.b128 B128_src, {%1, %2}; \n\t"
      "st.global.L1::evict_first.b128 [%0], B128_src; \n"
      "}"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"((*reinterpret_cast<longlong2*>(&src)).x),
        "l"((*reinterpret_cast<longlong2*>(&src)).y)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_first_is_not_supported_before_SM_70__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// st.space.L1::evict_first.v4.b64 [addr], src; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline void st_L1_evict_first(
  cuda_ptx::space_global_t,
  B256* addr,
  B256 src);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_st_L1_evict_first_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline void st_L1_evict_first(
  cuda_ptx::space_global_t,
  B256* addr,
  B256 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    asm (
      "st.global.L1::evict_first.v4.b64 [%0], {%1, %2, %3, %4};"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).x),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).y),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).z),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).w)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_first_is_not_supported_before_SM_100__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// st.space.L1::evict_first.L2::cache_hint.b8 [addr], src, cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void st_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  B8* addr,
  B8 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void st_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  B8* addr,
  B8 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "st.global.L1::evict_first.L2::cache_hint.b8 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "r"(__b8_as_u32(src)),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::evict_first.L2::cache_hint.b16 [addr], src, cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void st_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  B16* addr,
  B16 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void st_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  B16* addr,
  B16 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "st.global.L1::evict_first.L2::cache_hint.b16 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "h"(/*as_b16*/ *reinterpret_cast<const std::int16_t*>(&src)),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::evict_first.L2::cache_hint.b32 [addr], src, cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void st_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  B32* addr,
  B32 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void st_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  B32* addr,
  B32 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "st.global.L1::evict_first.L2::cache_hint.b32 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&src)),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::evict_first.L2::cache_hint.b64 [addr], src, cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void st_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  B64* addr,
  B64 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void st_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  B64* addr,
  B64 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "st.global.L1::evict_first.L2::cache_hint.b64 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&src)),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::evict_first.L2::cache_hint.b128 [addr], src, cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline void st_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  B128* addr,
  B128 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_st_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline void st_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  B128* addr,
  B128 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "{\n\t"
      ".reg .b128 B128_src; \n\t"
      "mov.b128 B128_src, {%1, %2}; \n\t"
      "st.global.L1::evict_first.L2::cache_hint.b128 [%0], B128_src, %3; \n"
      "}"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"((*reinterpret_cast<longlong2*>(&src)).x),
        "l"((*reinterpret_cast<longlong2*>(&src)).y),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// st.space.L1::evict_first.L2::cache_hint.v4.b64 [addr], src, cache_policy; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline void st_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  B256* addr,
  B256 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_st_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline void st_L1_evict_first_L2_cache_hint(
  cuda_ptx::space_global_t,
  B256* addr,
  B256 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    asm (
      "st.global.L1::evict_first.L2::cache_hint.v4.b64 [%0], {%1, %2, %3, %4}, %5;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).x),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).y),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).z),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).w),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_first_L2_cache_hint_is_not_supported_before_SM_100__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// st.space.L1::evict_last.b8 [addr], src; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void st_L1_evict_last(
  cuda_ptx::space_global_t,
  B8* addr,
  B8 src);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_evict_last_is_not_supported_before_SM_70__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void st_L1_evict_last(
  cuda_ptx::space_global_t,
  B8* addr,
  B8 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    asm (
      "st.global.L1::evict_last.b8 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "r"(__b8_as_u32(src))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_last_is_not_supported_before_SM_70__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::evict_last.b16 [addr], src; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void st_L1_evict_last(
  cuda_ptx::space_global_t,
  B16* addr,
  B16 src);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_evict_last_is_not_supported_before_SM_70__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void st_L1_evict_last(
  cuda_ptx::space_global_t,
  B16* addr,
  B16 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    asm (
      "st.global.L1::evict_last.b16 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "h"(/*as_b16*/ *reinterpret_cast<const std::int16_t*>(&src))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_last_is_not_supported_before_SM_70__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::evict_last.b32 [addr], src; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void st_L1_evict_last(
  cuda_ptx::space_global_t,
  B32* addr,
  B32 src);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_evict_last_is_not_supported_before_SM_70__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void st_L1_evict_last(
  cuda_ptx::space_global_t,
  B32* addr,
  B32 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    asm (
      "st.global.L1::evict_last.b32 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&src))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_last_is_not_supported_before_SM_70__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::evict_last.b64 [addr], src; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void st_L1_evict_last(
  cuda_ptx::space_global_t,
  B64* addr,
  B64 src);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_evict_last_is_not_supported_before_SM_70__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void st_L1_evict_last(
  cuda_ptx::space_global_t,
  B64* addr,
  B64 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    asm (
      "st.global.L1::evict_last.b64 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&src))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_last_is_not_supported_before_SM_70__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::evict_last.b128 [addr], src; // PTX ISA 83, SM_70
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline void st_L1_evict_last(
  cuda_ptx::space_global_t,
  B128* addr,
  B128 src);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_st_L1_evict_last_is_not_supported_before_SM_70__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline void st_L1_evict_last(
  cuda_ptx::space_global_t,
  B128* addr,
  B128 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    asm (
      "{\n\t"
      ".reg .b128 B128_src; \n\t"
      "mov.b128 B128_src, {%1, %2}; \n\t"
      "st.global.L1::evict_last.b128 [%0], B128_src; \n"
      "}"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"((*reinterpret_cast<longlong2*>(&src)).x),
        "l"((*reinterpret_cast<longlong2*>(&src)).y)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_last_is_not_supported_before_SM_70__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// st.space.L1::evict_last.v4.b64 [addr], src; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline void st_L1_evict_last(
  cuda_ptx::space_global_t,
  B256* addr,
  B256 src);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_st_L1_evict_last_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline void st_L1_evict_last(
  cuda_ptx::space_global_t,
  B256* addr,
  B256 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    asm (
      "st.global.L1::evict_last.v4.b64 [%0], {%1, %2, %3, %4};"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).x),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).y),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).z),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).w)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_last_is_not_supported_before_SM_100__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// st.space.L1::evict_last.L2::cache_hint.b8 [addr], src, cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void st_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  B8* addr,
  B8 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void st_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  B8* addr,
  B8 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "st.global.L1::evict_last.L2::cache_hint.b8 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "r"(__b8_as_u32(src)),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::evict_last.L2::cache_hint.b16 [addr], src, cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void st_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  B16* addr,
  B16 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void st_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  B16* addr,
  B16 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "st.global.L1::evict_last.L2::cache_hint.b16 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "h"(/*as_b16*/ *reinterpret_cast<const std::int16_t*>(&src)),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::evict_last.L2::cache_hint.b32 [addr], src, cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void st_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  B32* addr,
  B32 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void st_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  B32* addr,
  B32 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "st.global.L1::evict_last.L2::cache_hint.b32 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&src)),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::evict_last.L2::cache_hint.b64 [addr], src, cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void st_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  B64* addr,
  B64 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void st_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  B64* addr,
  B64 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "st.global.L1::evict_last.L2::cache_hint.b64 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&src)),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::evict_last.L2::cache_hint.b128 [addr], src, cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline void st_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  B128* addr,
  B128 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_st_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline void st_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  B128* addr,
  B128 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "{\n\t"
      ".reg .b128 B128_src; \n\t"
      "mov.b128 B128_src, {%1, %2}; \n\t"
      "st.global.L1::evict_last.L2::cache_hint.b128 [%0], B128_src, %3; \n"
      "}"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"((*reinterpret_cast<longlong2*>(&src)).x),
        "l"((*reinterpret_cast<longlong2*>(&src)).y),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// st.space.L1::evict_last.L2::cache_hint.v4.b64 [addr], src, cache_policy; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline void st_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  B256* addr,
  B256 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_st_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline void st_L1_evict_last_L2_cache_hint(
  cuda_ptx::space_global_t,
  B256* addr,
  B256 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    asm (
      "st.global.L1::evict_last.L2::cache_hint.v4.b64 [%0], {%1, %2, %3, %4}, %5;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).x),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).y),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).z),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).w),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_evict_last_L2_cache_hint_is_not_supported_before_SM_100__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// st.space.L1::no_allocate.b8 [addr], src; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void st_L1_no_allocate(
  cuda_ptx::space_global_t,
  B8* addr,
  B8 src);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_no_allocate_is_not_supported_before_SM_70__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void st_L1_no_allocate(
  cuda_ptx::space_global_t,
  B8* addr,
  B8 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    asm (
      "st.global.L1::no_allocate.b8 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "r"(__b8_as_u32(src))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_no_allocate_is_not_supported_before_SM_70__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::no_allocate.b16 [addr], src; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void st_L1_no_allocate(
  cuda_ptx::space_global_t,
  B16* addr,
  B16 src);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_no_allocate_is_not_supported_before_SM_70__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void st_L1_no_allocate(
  cuda_ptx::space_global_t,
  B16* addr,
  B16 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    asm (
      "st.global.L1::no_allocate.b16 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "h"(/*as_b16*/ *reinterpret_cast<const std::int16_t*>(&src))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_no_allocate_is_not_supported_before_SM_70__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::no_allocate.b32 [addr], src; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void st_L1_no_allocate(
  cuda_ptx::space_global_t,
  B32* addr,
  B32 src);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_no_allocate_is_not_supported_before_SM_70__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void st_L1_no_allocate(
  cuda_ptx::space_global_t,
  B32* addr,
  B32 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    asm (
      "st.global.L1::no_allocate.b32 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&src))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_no_allocate_is_not_supported_before_SM_70__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::no_allocate.b64 [addr], src; // PTX ISA 74, SM_70
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void st_L1_no_allocate(
  cuda_ptx::space_global_t,
  B64* addr,
  B64 src);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_no_allocate_is_not_supported_before_SM_70__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void st_L1_no_allocate(
  cuda_ptx::space_global_t,
  B64* addr,
  B64 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    asm (
      "st.global.L1::no_allocate.b64 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&src))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_no_allocate_is_not_supported_before_SM_70__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::no_allocate.b128 [addr], src; // PTX ISA 83, SM_70
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline void st_L1_no_allocate(
  cuda_ptx::space_global_t,
  B128* addr,
  B128 src);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_st_L1_no_allocate_is_not_supported_before_SM_70__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline void st_L1_no_allocate(
  cuda_ptx::space_global_t,
  B128* addr,
  B128 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 700
    asm (
      "{\n\t"
      ".reg .b128 B128_src; \n\t"
      "mov.b128 B128_src, {%1, %2}; \n\t"
      "st.global.L1::no_allocate.b128 [%0], B128_src; \n"
      "}"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"((*reinterpret_cast<longlong2*>(&src)).x),
        "l"((*reinterpret_cast<longlong2*>(&src)).y)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_no_allocate_is_not_supported_before_SM_70__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// st.space.L1::no_allocate.v4.b64 [addr], src; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline void st_L1_no_allocate(
  cuda_ptx::space_global_t,
  B256* addr,
  B256 src);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_st_L1_no_allocate_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline void st_L1_no_allocate(
  cuda_ptx::space_global_t,
  B256* addr,
  B256 src)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    asm (
      "st.global.L1::no_allocate.v4.b64 [%0], {%1, %2, %3, %4};"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).x),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).y),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).z),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).w)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_no_allocate_is_not_supported_before_SM_100__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// st.space.L1::no_allocate.L2::cache_hint.b8 [addr], src, cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void st_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  B8* addr,
  B8 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void st_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  B8* addr,
  B8 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B8) == 1, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "st.global.L1::no_allocate.L2::cache_hint.b8 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "r"(__b8_as_u32(src)),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::no_allocate.L2::cache_hint.b16 [addr], src, cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void st_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  B16* addr,
  B16 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void st_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  B16* addr,
  B16 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B16) == 2, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "st.global.L1::no_allocate.L2::cache_hint.b16 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "h"(/*as_b16*/ *reinterpret_cast<const std::int16_t*>(&src)),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::no_allocate.L2::cache_hint.b32 [addr], src, cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void st_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  B32* addr,
  B32 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void st_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  B32* addr,
  B32 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "st.global.L1::no_allocate.L2::cache_hint.b32 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&src)),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::no_allocate.L2::cache_hint.b64 [addr], src, cache_policy; // PTX ISA 74, SM_80
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void st_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  B64* addr,
  B64 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 740
extern "C" __device__ void __cuda_ptx_st_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void st_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  B64* addr,
  B64 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "st.global.L1::no_allocate.L2::cache_hint.b64 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&src)),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 740

/*
// st.space.L1::no_allocate.L2::cache_hint.b128 [addr], src, cache_policy; // PTX ISA 83, SM_80
// .space     = { .global }
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline void st_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  B128* addr,
  B128 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_st_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
template <typename B128, std::enable_if_t<sizeof(B128) == 16, bool> = true>
__device__ static inline void st_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  B128* addr,
  B128 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B128) == 16, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 800
    asm (
      "{\n\t"
      ".reg .b128 B128_src; \n\t"
      "mov.b128 B128_src, {%1, %2}; \n\t"
      "st.global.L1::no_allocate.L2::cache_hint.b128 [%0], B128_src, %3; \n"
      "}"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"((*reinterpret_cast<longlong2*>(&src)).x),
        "l"((*reinterpret_cast<longlong2*>(&src)).y),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_80__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// st.space.L1::no_allocate.L2::cache_hint.v4.b64 [addr], src, cache_policy; // PTX ISA 88, SM_100
// .space     = { .global }
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline void st_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  B256* addr,
  B256 src,
  std::uint64_t cache_policy);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_st_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_100__();
template <typename B256, std::enable_if_t<sizeof(B256) == 32, bool> = true>
__device__ static inline void st_L1_no_allocate_L2_cache_hint(
  cuda_ptx::space_global_t,
  B256* addr,
  B256 src,
  std::uint64_t cache_policy)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B256) == 32, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    asm (
      "st.global.L1::no_allocate.L2::cache_hint.v4.b64 [%0], {%1, %2, %3, %4}, %5;"
      :
      : "l"(__as_ptr_gmem(addr)),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).x),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).y),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).z),
        "l"((*reinterpret_cast<cuda_ptx::longlong4_32a*>(&src)).w),
        "l"(cache_policy)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_L1_no_allocate_L2_cache_hint_is_not_supported_before_SM_100__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

} // namespace cuda_ptx

#endif // INSTRUCTIONS_ST_H

// End of ./instructions/st.h
// Start of ./instructions/st_async.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_ST_ASYNC_H
#define INSTRUCTIONS_ST_ASYNC_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/st_async.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.type [addr], value, [remote_bar];    // 1.  PTX ISA 81, SM_90
// .type      = { .b32, .b64 }
template <typename Type>
__device__ static inline void st_async(
  Type* addr,
  const Type& value,
  std::uint64_t* remote_bar);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_st_async_is_not_supported_before_SM_90__();
template <typename Type>
__device__ static inline void st_async(
  Type* addr,
  const Type& value,
  std::uint64_t* remote_bar)
{
  static_assert(sizeof(Type) == 4 || sizeof(Type) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sizeof(Type) == 4) {
      asm (
        "st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.b32 [%0], %1, [%2];    // 1. "
        :
        : "r"(__as_ptr_remote_dsmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&value)),
          "r"(__as_ptr_remote_dsmem(remote_bar))
        : "memory"
      );
    } else if constexpr (sizeof(Type) == 8) {
      asm (
        "st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.b64 [%0], %1, [%2];    // 1. "
        :
        : "r"(__as_ptr_remote_dsmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&value)),
          "r"(__as_ptr_remote_dsmem(remote_bar))
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_async_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.v2.type [addr], value, [remote_bar]; // 2.  PTX ISA 81, SM_90
// .type      = { .b32, .b64 }
template <typename Type>
__device__ static inline void st_async(
  Type* addr,
  const Type (&value)[2],
  std::uint64_t* remote_bar);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_st_async_is_not_supported_before_SM_90__();
template <typename Type>
__device__ static inline void st_async(
  Type* addr,
  const Type (&value)[2],
  std::uint64_t* remote_bar)
{
  static_assert(sizeof(Type) == 4 || sizeof(Type) == 8, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (sizeof(Type) == 4) {
      asm (
        "st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.v2.b32 [%0], {%1, %2}, [%3]; // 2. "
        :
        : "r"(__as_ptr_remote_dsmem(addr)),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&value[0])),
          "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&value[1])),
          "r"(__as_ptr_remote_dsmem(remote_bar))
        : "memory"
      );
    } else if constexpr (sizeof(Type) == 8) {
      asm (
        "st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.v2.b64 [%0], {%1, %2}, [%3]; // 2. "
        :
        : "r"(__as_ptr_remote_dsmem(addr)),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&value[0])),
          "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&value[1])),
          "r"(__as_ptr_remote_dsmem(remote_bar))
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_async_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

/*
// st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.v4.b32 [addr], value, [remote_bar];    // 3.  PTX ISA 81, SM_90
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void st_async(
  B32* addr,
  const B32 (&value)[4],
  std::uint64_t* remote_bar);
*/
#if __libcuda_ptx_isa >= 810
extern "C" __device__ void __cuda_ptx_st_async_is_not_supported_before_SM_90__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void st_async(
  B32* addr,
  const B32 (&value)[4],
  std::uint64_t* remote_bar)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm (
      "st.async.weak.shared::cluster.mbarrier::complete_tx::bytes.v4.b32 [%0], {%1, %2, %3, %4}, [%5];    // 3. "
      :
      : "r"(__as_ptr_remote_dsmem(addr)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&value[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&value[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&value[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&value[3])),
        "r"(__as_ptr_remote_dsmem(remote_bar))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_async_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 810

} // namespace cuda_ptx

#endif // INSTRUCTIONS_ST_ASYNC_H

// End of ./instructions/st_async.h
// Start of ./instructions/st_bulk.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_ST_BULK_H
#define INSTRUCTIONS_ST_BULK_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/st_bulk.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// st.bulk.weak.shared::cta [addr], size, initval; // PTX ISA 86, SM_100
template <int N32>
__device__ static inline void st_bulk(
  void* addr,
  std::uint64_t size,
  cuda_ptx::n32_t<N32> initval);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_st_bulk_is_not_supported_before_SM_100__();
template <int N32>
__device__ static inline void st_bulk(
  void* addr,
  std::uint64_t size,
  cuda_ptx::n32_t<N32> initval)
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 1000
    asm (
      "st.bulk.weak.shared::cta [%0], %1, %2;"
      :
      : "r"(__as_ptr_smem(addr)),
        "l"(size),
        "n"(initval.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_st_bulk_is_not_supported_before_SM_100__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

} // namespace cuda_ptx

#endif // INSTRUCTIONS_ST_BULK_H

// End of ./instructions/st_bulk.h
// Start of ./instructions/stmatrix.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_STMATRIX_H
#define INSTRUCTIONS_STMATRIX_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/stmatrix.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// stmatrix.sync.aligned.m8n8.x1.shared.b16 [gmem_ptr], input; // PTX ISA 78, SM_90
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void stmatrix_m8n8(
  B16* gmem_ptr,
  const std::uint32_t (&input)[1]);
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_stmatrix_m8n8_is_not_supported_before_SM_90__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void stmatrix_m8n8(
  B16* gmem_ptr,
  const std::uint32_t (&input)[1])
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "stmatrix.sync.aligned.m8n8.x1.shared.b16 [%0], {%1};"
      :
      : "r"(__as_ptr_smem(gmem_ptr)),
        "r"(input[0])
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_stmatrix_m8n8_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// stmatrix.sync.aligned.m8n8.x2.shared.b16 [gmem_ptr], input; // PTX ISA 78, SM_90
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void stmatrix_m8n8(
  B16* gmem_ptr,
  const std::uint32_t (&input)[2]);
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_stmatrix_m8n8_is_not_supported_before_SM_90__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void stmatrix_m8n8(
  B16* gmem_ptr,
  const std::uint32_t (&input)[2])
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "stmatrix.sync.aligned.m8n8.x2.shared.b16 [%0], {%1, %2};"
      :
      : "r"(__as_ptr_smem(gmem_ptr)),
        "r"(input[0]),
        "r"(input[1])
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_stmatrix_m8n8_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// stmatrix.sync.aligned.m8n8.x4.shared.b16 [gmem_ptr], input; // PTX ISA 78, SM_90
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void stmatrix_m8n8(
  B16* gmem_ptr,
  const std::uint32_t (&input)[4]);
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_stmatrix_m8n8_is_not_supported_before_SM_90__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void stmatrix_m8n8(
  B16* gmem_ptr,
  const std::uint32_t (&input)[4])
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "stmatrix.sync.aligned.m8n8.x4.shared.b16 [%0], {%1, %2, %3, %4};"
      :
      : "r"(__as_ptr_smem(gmem_ptr)),
        "r"(input[0]),
        "r"(input[1]),
        "r"(input[2]),
        "r"(input[3])
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_stmatrix_m8n8_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// stmatrix.sync.aligned.m8n8.x1.trans.shared.b16 [gmem_ptr], input; // PTX ISA 78, SM_90
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void stmatrix_m8n8_trans(
  B16* gmem_ptr,
  const std::uint32_t (&input)[1]);
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_stmatrix_m8n8_trans_is_not_supported_before_SM_90__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void stmatrix_m8n8_trans(
  B16* gmem_ptr,
  const std::uint32_t (&input)[1])
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "stmatrix.sync.aligned.m8n8.x1.trans.shared.b16 [%0], {%1};"
      :
      : "r"(__as_ptr_smem(gmem_ptr)),
        "r"(input[0])
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_stmatrix_m8n8_trans_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// stmatrix.sync.aligned.m8n8.x2.trans.shared.b16 [gmem_ptr], input; // PTX ISA 78, SM_90
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void stmatrix_m8n8_trans(
  B16* gmem_ptr,
  const std::uint32_t (&input)[2]);
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_stmatrix_m8n8_trans_is_not_supported_before_SM_90__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void stmatrix_m8n8_trans(
  B16* gmem_ptr,
  const std::uint32_t (&input)[2])
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "stmatrix.sync.aligned.m8n8.x2.trans.shared.b16 [%0], {%1, %2};"
      :
      : "r"(__as_ptr_smem(gmem_ptr)),
        "r"(input[0]),
        "r"(input[1])
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_stmatrix_m8n8_trans_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [gmem_ptr], input; // PTX ISA 78, SM_90
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void stmatrix_m8n8_trans(
  B16* gmem_ptr,
  const std::uint32_t (&input)[4]);
*/
#if __libcuda_ptx_isa >= 780
extern "C" __device__ void __cuda_ptx_stmatrix_m8n8_trans_is_not_supported_before_SM_90__();
template <typename B16, std::enable_if_t<sizeof(B16) == 2, bool> = true>
__device__ static inline void stmatrix_m8n8_trans(
  B16* gmem_ptr,
  const std::uint32_t (&input)[4])
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    asm volatile (
      "stmatrix.sync.aligned.m8n8.x4.trans.shared.b16 [%0], {%1, %2, %3, %4};"
      :
      : "r"(__as_ptr_smem(gmem_ptr)),
        "r"(input[0]),
        "r"(input[1]),
        "r"(input[2]),
        "r"(input[3])
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_stmatrix_m8n8_trans_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 780

/*
// stmatrix.sync.aligned.m16n8.x1.trans.shared.b8 [gmem_ptr], input; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void stmatrix_m16n8_trans(
  B8* gmem_ptr,
  const std::uint32_t (&input)[1]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_stmatrix_m16n8_trans_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void stmatrix_m16n8_trans(
  B8* gmem_ptr,
  const std::uint32_t (&input)[1])
{
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm volatile (
      "stmatrix.sync.aligned.m16n8.x1.trans.shared.b8 [%0], {%1};"
      :
      : "r"(__as_ptr_smem(gmem_ptr)),
        "r"(input[0])
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_stmatrix_m16n8_trans_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// stmatrix.sync.aligned.m16n8.x2.trans.shared.b8 [gmem_ptr], input; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void stmatrix_m16n8_trans(
  B8* gmem_ptr,
  const std::uint32_t (&input)[2]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_stmatrix_m16n8_trans_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void stmatrix_m16n8_trans(
  B8* gmem_ptr,
  const std::uint32_t (&input)[2])
{
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm volatile (
      "stmatrix.sync.aligned.m16n8.x2.trans.shared.b8 [%0], {%1, %2};"
      :
      : "r"(__as_ptr_smem(gmem_ptr)),
        "r"(input[0]),
        "r"(input[1])
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_stmatrix_m16n8_trans_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// stmatrix.sync.aligned.m16n8.x4.trans.shared.b8 [gmem_ptr], input; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void stmatrix_m16n8_trans(
  B8* gmem_ptr,
  const std::uint32_t (&input)[4]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_stmatrix_m16n8_trans_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <typename B8, std::enable_if_t<sizeof(B8) == 1, bool> = true>
__device__ static inline void stmatrix_m16n8_trans(
  B8* gmem_ptr,
  const std::uint32_t (&input)[4])
{
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm volatile (
      "stmatrix.sync.aligned.m16n8.x4.trans.shared.b8 [%0], {%1, %2, %3, %4};"
      :
      : "r"(__as_ptr_smem(gmem_ptr)),
        "r"(input[0]),
        "r"(input[1]),
        "r"(input[2]),
        "r"(input[3])
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_stmatrix_m16n8_trans_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

} // namespace cuda_ptx

#endif // INSTRUCTIONS_STMATRIX_H

// End of ./instructions/stmatrix.h
// Start of ./instructions/tcgen05_alloc.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_TCGEN05_ALLOC_H
#define INSTRUCTIONS_TCGEN05_ALLOC_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/tcgen05_alloc.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// tcgen05.alloc.cta_group.sync.aligned.shared::cta.b32 [dst], nCols; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_alloc(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t* dst,
  const std::uint32_t& nCols);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_alloc_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_alloc(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t* dst,
  const std::uint32_t& nCols)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;"
        :
        : "r"(__as_ptr_smem(dst)),
          "r"(nCols)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "tcgen05.alloc.cta_group::2.sync.aligned.shared::cta.b32 [%0], %1;"
        :
        : "r"(__as_ptr_smem(dst)),
          "r"(nCols)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_alloc_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.dealloc.cta_group.sync.aligned.b32 taddr, nCols; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_dealloc(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  const std::uint32_t& nCols);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_dealloc_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_dealloc(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  const std::uint32_t& nCols)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;"
        :
        : "r"(taddr),
          "r"(nCols)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "tcgen05.dealloc.cta_group::2.sync.aligned.b32 %0, %1;"
        :
        : "r"(taddr),
          "r"(nCols)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_dealloc_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860



/*
// tcgen05.relinquish_alloc_permit.cta_group.sync.aligned; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_relinquish_alloc_permit(
  cuda_ptx::cta_group_t<Cta_Group> cta_group);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_relinquish_alloc_permit_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_relinquish_alloc_permit(
  cuda_ptx::cta_group_t<Cta_Group> cta_group)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;"
        :
        :
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "tcgen05.relinquish_alloc_permit.cta_group::2.sync.aligned;"
        :
        :
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_relinquish_alloc_permit_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

} // namespace cuda_ptx

#endif // INSTRUCTIONS_TCGEN05_ALLOC_H

// End of ./instructions/tcgen05_alloc.h
// Start of ./instructions/tcgen05_commit.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_TCGEN05_COMMIT_H
#define INSTRUCTIONS_TCGEN05_COMMIT_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/tcgen05_commit.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// tcgen05.commit.cta_group.mbarrier::arrive::one.shared::cluster.b64 [smem_bar]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_commit(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint64_t* smem_bar);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_commit_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_commit(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint64_t* smem_bar)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.b64 [%0];"
        :
        : "r"(__as_ptr_dsmem(smem_bar))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.b64 [%0];"
        :
        : "r"(__as_ptr_dsmem(smem_bar))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_commit_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.commit.cta_group.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [smem_bar], ctaMask; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_commit_multicast(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint64_t* smem_bar,
  std::uint16_t ctaMask);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_commit_multicast_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_commit_multicast(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint64_t* smem_bar,
  std::uint16_t ctaMask)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "tcgen05.commit.cta_group::1.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%0], %1;"
        :
        : "r"(__as_ptr_dsmem(smem_bar)),
          "h"(ctaMask)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "tcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [%0], %1;"
        :
        : "r"(__as_ptr_dsmem(smem_bar)),
          "h"(ctaMask)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_commit_multicast_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

} // namespace cuda_ptx

#endif // INSTRUCTIONS_TCGEN05_COMMIT_H

// End of ./instructions/tcgen05_commit.h
// Start of ./instructions/tcgen05_cp.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_TCGEN05_CP_H
#define INSTRUCTIONS_TCGEN05_CP_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/tcgen05_cp.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// tcgen05.cp.cta_group.128x256b [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_128x256b(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_cp_128x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_128x256b(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "tcgen05.cp.cta_group::1.128x256b [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "tcgen05.cp.cta_group::2.128x256b [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_cp_128x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.cp.cta_group.4x256b [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_4x256b(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_cp_4x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_4x256b(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "tcgen05.cp.cta_group::1.4x256b [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "tcgen05.cp.cta_group::2.4x256b [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_cp_4x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.cp.cta_group.128x128b [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_128x128b(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_cp_128x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_128x128b(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "tcgen05.cp.cta_group::1.128x128b [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "tcgen05.cp.cta_group::2.128x128b [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_cp_128x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.cp.cta_group.64x128b.warpx2::02_13 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_64x128b_warpx2_02_13(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_cp_64x128b_warpx2_02_13_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_64x128b_warpx2_02_13(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "tcgen05.cp.cta_group::1.64x128b.warpx2::02_13 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "tcgen05.cp.cta_group::2.64x128b.warpx2::02_13 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_cp_64x128b_warpx2_02_13_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.cp.cta_group.64x128b.warpx2::01_23 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_64x128b_warpx2_01_23(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_cp_64x128b_warpx2_01_23_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_64x128b_warpx2_01_23(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "tcgen05.cp.cta_group::1.64x128b.warpx2::01_23 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "tcgen05.cp.cta_group::2.64x128b.warpx2::01_23 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_cp_64x128b_warpx2_01_23_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.cp.cta_group.32x128b.warpx4 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_32x128b_warpx4(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_cp_32x128b_warpx4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_32x128b_warpx4(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "tcgen05.cp.cta_group::1.32x128b.warpx4 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "tcgen05.cp.cta_group::2.32x128b.warpx4 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_cp_32x128b_warpx4_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.cp.cta_group.128x256b.b8x16.b6x16_p32 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_128x256b_b8x16_b6x16_p32(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_cp_128x256b_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_128x256b_b8x16_b6x16_p32(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "tcgen05.cp.cta_group::1.128x256b.b8x16.b6x16_p32 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "tcgen05.cp.cta_group::2.128x256b.b8x16.b6x16_p32 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_cp_128x256b_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.cp.cta_group.4x256b.b8x16.b6x16_p32 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_4x256b_b8x16_b6x16_p32(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_cp_4x256b_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_4x256b_b8x16_b6x16_p32(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "tcgen05.cp.cta_group::1.4x256b.b8x16.b6x16_p32 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "tcgen05.cp.cta_group::2.4x256b.b8x16.b6x16_p32 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_cp_4x256b_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.cp.cta_group.128x128b.b8x16.b6x16_p32 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_128x128b_b8x16_b6x16_p32(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_cp_128x128b_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_128x128b_b8x16_b6x16_p32(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "tcgen05.cp.cta_group::1.128x128b.b8x16.b6x16_p32 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "tcgen05.cp.cta_group::2.128x128b.b8x16.b6x16_p32 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_cp_128x128b_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.cp.cta_group.64x128b.warpx2::02_13.b8x16.b6x16_p32 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_64x128b_warpx2_02_13_b8x16_b6x16_p32(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_cp_64x128b_warpx2_02_13_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_64x128b_warpx2_02_13_b8x16_b6x16_p32(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "tcgen05.cp.cta_group::1.64x128b.warpx2::02_13.b8x16.b6x16_p32 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "tcgen05.cp.cta_group::2.64x128b.warpx2::02_13.b8x16.b6x16_p32 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_cp_64x128b_warpx2_02_13_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.cp.cta_group.64x128b.warpx2::01_23.b8x16.b6x16_p32 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_64x128b_warpx2_01_23_b8x16_b6x16_p32(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_cp_64x128b_warpx2_01_23_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_64x128b_warpx2_01_23_b8x16_b6x16_p32(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "tcgen05.cp.cta_group::1.64x128b.warpx2::01_23.b8x16.b6x16_p32 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "tcgen05.cp.cta_group::2.64x128b.warpx2::01_23.b8x16.b6x16_p32 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_cp_64x128b_warpx2_01_23_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.cp.cta_group.32x128b.warpx4.b8x16.b6x16_p32 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_32x128b_warpx4_b8x16_b6x16_p32(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_cp_32x128b_warpx4_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_32x128b_warpx4_b8x16_b6x16_p32(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "tcgen05.cp.cta_group::1.32x128b.warpx4.b8x16.b6x16_p32 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "tcgen05.cp.cta_group::2.32x128b.warpx4.b8x16.b6x16_p32 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_cp_32x128b_warpx4_b8x16_b6x16_p32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.cp.cta_group.128x256b.b8x16.b4x16_p64 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_128x256b_b8x16_b4x16_p64(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_cp_128x256b_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_128x256b_b8x16_b4x16_p64(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "tcgen05.cp.cta_group::1.128x256b.b8x16.b4x16_p64 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "tcgen05.cp.cta_group::2.128x256b.b8x16.b4x16_p64 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_cp_128x256b_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.cp.cta_group.4x256b.b8x16.b4x16_p64 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_4x256b_b8x16_b4x16_p64(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_cp_4x256b_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_4x256b_b8x16_b4x16_p64(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "tcgen05.cp.cta_group::1.4x256b.b8x16.b4x16_p64 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "tcgen05.cp.cta_group::2.4x256b.b8x16.b4x16_p64 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_cp_4x256b_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.cp.cta_group.128x128b.b8x16.b4x16_p64 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_128x128b_b8x16_b4x16_p64(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_cp_128x128b_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_128x128b_b8x16_b4x16_p64(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "tcgen05.cp.cta_group::1.128x128b.b8x16.b4x16_p64 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "tcgen05.cp.cta_group::2.128x128b.b8x16.b4x16_p64 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_cp_128x128b_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.cp.cta_group.64x128b.warpx2::02_13.b8x16.b4x16_p64 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_64x128b_warpx2_02_13_b8x16_b4x16_p64(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_cp_64x128b_warpx2_02_13_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_64x128b_warpx2_02_13_b8x16_b4x16_p64(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "tcgen05.cp.cta_group::1.64x128b.warpx2::02_13.b8x16.b4x16_p64 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "tcgen05.cp.cta_group::2.64x128b.warpx2::02_13.b8x16.b4x16_p64 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_cp_64x128b_warpx2_02_13_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.cp.cta_group.64x128b.warpx2::01_23.b8x16.b4x16_p64 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_64x128b_warpx2_01_23_b8x16_b4x16_p64(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_cp_64x128b_warpx2_01_23_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_64x128b_warpx2_01_23_b8x16_b4x16_p64(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "tcgen05.cp.cta_group::1.64x128b.warpx2::01_23.b8x16.b4x16_p64 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "tcgen05.cp.cta_group::2.64x128b.warpx2::01_23.b8x16.b4x16_p64 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_cp_64x128b_warpx2_01_23_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.cp.cta_group.32x128b.warpx4.b8x16.b4x16_p64 [taddr], s_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_32x128b_warpx4_b8x16_b4x16_p64(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_cp_32x128b_warpx4_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_cp_32x128b_warpx4_b8x16_b4x16_p64(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr,
  std::uint64_t s_desc)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm (
        "tcgen05.cp.cta_group::1.32x128b.warpx4.b8x16.b4x16_p64 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm (
        "tcgen05.cp.cta_group::2.32x128b.warpx4.b8x16.b4x16_p64 [%0], %1;"
        :
        : "r"(taddr),
          "l"(s_desc)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_cp_32x128b_warpx4_b8x16_b4x16_p64_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

} // namespace cuda_ptx

#endif // INSTRUCTIONS_TCGEN05_CP_H

// End of ./instructions/tcgen05_cp.h
// Start of ./instructions/tcgen05_fence.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_TCGEN05_FENCE_H
#define INSTRUCTIONS_TCGEN05_FENCE_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/tcgen05_fence.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// tcgen05.fence::before_thread_sync; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename = void>
__device__ static inline void tcgen05_fence_before_thread_sync();
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_fence_before_thread_sync_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename = void>
__device__ static inline void tcgen05_fence_before_thread_sync()
{
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm volatile (
      "tcgen05.fence::before_thread_sync;"
      :
      :
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_fence_before_thread_sync_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.fence::after_thread_sync; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename = void>
__device__ static inline void tcgen05_fence_after_thread_sync();
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_fence_after_thread_sync_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename = void>
__device__ static inline void tcgen05_fence_after_thread_sync()
{
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm volatile (
      "tcgen05.fence::after_thread_sync;"
      :
      :
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_fence_after_thread_sync_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

} // namespace cuda_ptx

#endif // INSTRUCTIONS_TCGEN05_FENCE_H

// End of ./instructions/tcgen05_fence.h
// Start of ./instructions/tcgen05_ld.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_TCGEN05_LD_H
#define INSTRUCTIONS_TCGEN05_LD_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/tcgen05_ld.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// tcgen05.ld.sync.aligned.16x64b.x1.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b(
  B32 (&out)[1],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b(
  B32 (&out)[1],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x64b.x1.b32 {%0}, [%1];"
      : "=r"(out[0])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x64b.x1.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b_pack_16b(
  B32 (&out)[1],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x64b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b_pack_16b(
  B32 (&out)[1],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x64b.x1.pack::16b.b32 {%0}, [%1];"
      : "=r"(out[0])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x64b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x64b.x2.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b(
  B32 (&out)[2],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b(
  B32 (&out)[2],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x64b.x2.b32 {%0, %1}, [%2];"
      : "=r"(out[0]),
        "=r"(out[1])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x64b.x2.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b_pack_16b(
  B32 (&out)[2],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x64b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b_pack_16b(
  B32 (&out)[2],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x64b.x2.pack::16b.b32 {%0, %1}, [%2];"
      : "=r"(out[0]),
        "=r"(out[1])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x64b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x64b.x4.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b(
  B32 (&out)[4],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b(
  B32 (&out)[4],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x64b.x4.b32 {%0, %1, %2, %3}, [%4];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x64b.x4.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b_pack_16b(
  B32 (&out)[4],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x64b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b_pack_16b(
  B32 (&out)[4],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x64b.x4.pack::16b.b32 {%0, %1, %2, %3}, [%4];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x64b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x64b.x8.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b(
  B32 (&out)[8],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b(
  B32 (&out)[8],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x64b.x8.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x64b.x8.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b_pack_16b(
  B32 (&out)[8],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x64b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b_pack_16b(
  B32 (&out)[8],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x64b.x8.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x64b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x64b.x16.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b(
  B32 (&out)[16],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b(
  B32 (&out)[16],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x64b.x16.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x64b.x16.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b_pack_16b(
  B32 (&out)[16],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x64b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b_pack_16b(
  B32 (&out)[16],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x64b.x16.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x64b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x64b.x32.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b(
  B32 (&out)[32],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b(
  B32 (&out)[32],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x64b.x32.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, [%32];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x64b.x32.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b_pack_16b(
  B32 (&out)[32],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x64b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b_pack_16b(
  B32 (&out)[32],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x64b.x32.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, [%32];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x64b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x64b.x64.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b(
  B32 (&out)[64],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b(
  B32 (&out)[64],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x64b.x64.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, [%64];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x64b.x64.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b_pack_16b(
  B32 (&out)[64],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x64b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b_pack_16b(
  B32 (&out)[64],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x64b.x64.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, [%64];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x64b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x64b.x128.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b(
  B32 (&out)[128],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b(
  B32 (&out)[128],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x64b.x128.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, [%128];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63]),
        "=r"(out[64]),
        "=r"(out[65]),
        "=r"(out[66]),
        "=r"(out[67]),
        "=r"(out[68]),
        "=r"(out[69]),
        "=r"(out[70]),
        "=r"(out[71]),
        "=r"(out[72]),
        "=r"(out[73]),
        "=r"(out[74]),
        "=r"(out[75]),
        "=r"(out[76]),
        "=r"(out[77]),
        "=r"(out[78]),
        "=r"(out[79]),
        "=r"(out[80]),
        "=r"(out[81]),
        "=r"(out[82]),
        "=r"(out[83]),
        "=r"(out[84]),
        "=r"(out[85]),
        "=r"(out[86]),
        "=r"(out[87]),
        "=r"(out[88]),
        "=r"(out[89]),
        "=r"(out[90]),
        "=r"(out[91]),
        "=r"(out[92]),
        "=r"(out[93]),
        "=r"(out[94]),
        "=r"(out[95]),
        "=r"(out[96]),
        "=r"(out[97]),
        "=r"(out[98]),
        "=r"(out[99]),
        "=r"(out[100]),
        "=r"(out[101]),
        "=r"(out[102]),
        "=r"(out[103]),
        "=r"(out[104]),
        "=r"(out[105]),
        "=r"(out[106]),
        "=r"(out[107]),
        "=r"(out[108]),
        "=r"(out[109]),
        "=r"(out[110]),
        "=r"(out[111]),
        "=r"(out[112]),
        "=r"(out[113]),
        "=r"(out[114]),
        "=r"(out[115]),
        "=r"(out[116]),
        "=r"(out[117]),
        "=r"(out[118]),
        "=r"(out[119]),
        "=r"(out[120]),
        "=r"(out[121]),
        "=r"(out[122]),
        "=r"(out[123]),
        "=r"(out[124]),
        "=r"(out[125]),
        "=r"(out[126]),
        "=r"(out[127])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x64b.x128.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b_pack_16b(
  B32 (&out)[128],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x64b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x64b_pack_16b(
  B32 (&out)[128],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x64b.x128.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, [%128];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63]),
        "=r"(out[64]),
        "=r"(out[65]),
        "=r"(out[66]),
        "=r"(out[67]),
        "=r"(out[68]),
        "=r"(out[69]),
        "=r"(out[70]),
        "=r"(out[71]),
        "=r"(out[72]),
        "=r"(out[73]),
        "=r"(out[74]),
        "=r"(out[75]),
        "=r"(out[76]),
        "=r"(out[77]),
        "=r"(out[78]),
        "=r"(out[79]),
        "=r"(out[80]),
        "=r"(out[81]),
        "=r"(out[82]),
        "=r"(out[83]),
        "=r"(out[84]),
        "=r"(out[85]),
        "=r"(out[86]),
        "=r"(out[87]),
        "=r"(out[88]),
        "=r"(out[89]),
        "=r"(out[90]),
        "=r"(out[91]),
        "=r"(out[92]),
        "=r"(out[93]),
        "=r"(out[94]),
        "=r"(out[95]),
        "=r"(out[96]),
        "=r"(out[97]),
        "=r"(out[98]),
        "=r"(out[99]),
        "=r"(out[100]),
        "=r"(out[101]),
        "=r"(out[102]),
        "=r"(out[103]),
        "=r"(out[104]),
        "=r"(out[105]),
        "=r"(out[106]),
        "=r"(out[107]),
        "=r"(out[108]),
        "=r"(out[109]),
        "=r"(out[110]),
        "=r"(out[111]),
        "=r"(out[112]),
        "=r"(out[113]),
        "=r"(out[114]),
        "=r"(out[115]),
        "=r"(out[116]),
        "=r"(out[117]),
        "=r"(out[118]),
        "=r"(out[119]),
        "=r"(out[120]),
        "=r"(out[121]),
        "=r"(out[122]),
        "=r"(out[123]),
        "=r"(out[124]),
        "=r"(out[125]),
        "=r"(out[126]),
        "=r"(out[127])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x64b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x128b.x1.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b(
  B32 (&out)[2],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b(
  B32 (&out)[2],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x128b.x1.b32 {%0, %1}, [%2];"
      : "=r"(out[0]),
        "=r"(out[1])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x128b.x1.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b_pack_16b(
  B32 (&out)[2],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x128b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b_pack_16b(
  B32 (&out)[2],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x128b.x1.pack::16b.b32 {%0, %1}, [%2];"
      : "=r"(out[0]),
        "=r"(out[1])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x128b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x128b.x2.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b(
  B32 (&out)[4],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b(
  B32 (&out)[4],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x128b.x2.b32 {%0, %1, %2, %3}, [%4];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x128b.x2.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b_pack_16b(
  B32 (&out)[4],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x128b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b_pack_16b(
  B32 (&out)[4],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x128b.x2.pack::16b.b32 {%0, %1, %2, %3}, [%4];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x128b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x128b.x4.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b(
  B32 (&out)[8],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b(
  B32 (&out)[8],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x128b.x4.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x128b.x4.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b_pack_16b(
  B32 (&out)[8],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x128b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b_pack_16b(
  B32 (&out)[8],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x128b.x4.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x128b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x128b.x8.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b(
  B32 (&out)[16],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b(
  B32 (&out)[16],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x128b.x8.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x128b.x8.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b_pack_16b(
  B32 (&out)[16],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x128b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b_pack_16b(
  B32 (&out)[16],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x128b.x8.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x128b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x128b.x16.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b(
  B32 (&out)[32],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b(
  B32 (&out)[32],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x128b.x16.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, [%32];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x128b.x16.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b_pack_16b(
  B32 (&out)[32],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x128b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b_pack_16b(
  B32 (&out)[32],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x128b.x16.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, [%32];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x128b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x128b.x32.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b(
  B32 (&out)[64],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b(
  B32 (&out)[64],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x128b.x32.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, [%64];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x128b.x32.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b_pack_16b(
  B32 (&out)[64],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x128b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b_pack_16b(
  B32 (&out)[64],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x128b.x32.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, [%64];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x128b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x128b.x64.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b(
  B32 (&out)[128],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b(
  B32 (&out)[128],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x128b.x64.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, [%128];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63]),
        "=r"(out[64]),
        "=r"(out[65]),
        "=r"(out[66]),
        "=r"(out[67]),
        "=r"(out[68]),
        "=r"(out[69]),
        "=r"(out[70]),
        "=r"(out[71]),
        "=r"(out[72]),
        "=r"(out[73]),
        "=r"(out[74]),
        "=r"(out[75]),
        "=r"(out[76]),
        "=r"(out[77]),
        "=r"(out[78]),
        "=r"(out[79]),
        "=r"(out[80]),
        "=r"(out[81]),
        "=r"(out[82]),
        "=r"(out[83]),
        "=r"(out[84]),
        "=r"(out[85]),
        "=r"(out[86]),
        "=r"(out[87]),
        "=r"(out[88]),
        "=r"(out[89]),
        "=r"(out[90]),
        "=r"(out[91]),
        "=r"(out[92]),
        "=r"(out[93]),
        "=r"(out[94]),
        "=r"(out[95]),
        "=r"(out[96]),
        "=r"(out[97]),
        "=r"(out[98]),
        "=r"(out[99]),
        "=r"(out[100]),
        "=r"(out[101]),
        "=r"(out[102]),
        "=r"(out[103]),
        "=r"(out[104]),
        "=r"(out[105]),
        "=r"(out[106]),
        "=r"(out[107]),
        "=r"(out[108]),
        "=r"(out[109]),
        "=r"(out[110]),
        "=r"(out[111]),
        "=r"(out[112]),
        "=r"(out[113]),
        "=r"(out[114]),
        "=r"(out[115]),
        "=r"(out[116]),
        "=r"(out[117]),
        "=r"(out[118]),
        "=r"(out[119]),
        "=r"(out[120]),
        "=r"(out[121]),
        "=r"(out[122]),
        "=r"(out[123]),
        "=r"(out[124]),
        "=r"(out[125]),
        "=r"(out[126]),
        "=r"(out[127])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x128b.x64.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b_pack_16b(
  B32 (&out)[128],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x128b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x128b_pack_16b(
  B32 (&out)[128],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x128b.x64.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, [%128];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63]),
        "=r"(out[64]),
        "=r"(out[65]),
        "=r"(out[66]),
        "=r"(out[67]),
        "=r"(out[68]),
        "=r"(out[69]),
        "=r"(out[70]),
        "=r"(out[71]),
        "=r"(out[72]),
        "=r"(out[73]),
        "=r"(out[74]),
        "=r"(out[75]),
        "=r"(out[76]),
        "=r"(out[77]),
        "=r"(out[78]),
        "=r"(out[79]),
        "=r"(out[80]),
        "=r"(out[81]),
        "=r"(out[82]),
        "=r"(out[83]),
        "=r"(out[84]),
        "=r"(out[85]),
        "=r"(out[86]),
        "=r"(out[87]),
        "=r"(out[88]),
        "=r"(out[89]),
        "=r"(out[90]),
        "=r"(out[91]),
        "=r"(out[92]),
        "=r"(out[93]),
        "=r"(out[94]),
        "=r"(out[95]),
        "=r"(out[96]),
        "=r"(out[97]),
        "=r"(out[98]),
        "=r"(out[99]),
        "=r"(out[100]),
        "=r"(out[101]),
        "=r"(out[102]),
        "=r"(out[103]),
        "=r"(out[104]),
        "=r"(out[105]),
        "=r"(out[106]),
        "=r"(out[107]),
        "=r"(out[108]),
        "=r"(out[109]),
        "=r"(out[110]),
        "=r"(out[111]),
        "=r"(out[112]),
        "=r"(out[113]),
        "=r"(out[114]),
        "=r"(out[115]),
        "=r"(out[116]),
        "=r"(out[117]),
        "=r"(out[118]),
        "=r"(out[119]),
        "=r"(out[120]),
        "=r"(out[121]),
        "=r"(out[122]),
        "=r"(out[123]),
        "=r"(out[124]),
        "=r"(out[125]),
        "=r"(out[126]),
        "=r"(out[127])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x128b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x256b.x1.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b(
  B32 (&out)[4],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b(
  B32 (&out)[4],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x256b.x1.b32 {%0, %1, %2, %3}, [%4];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x256b.x1.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b_pack_16b(
  B32 (&out)[4],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x256b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b_pack_16b(
  B32 (&out)[4],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x256b.x1.pack::16b.b32 {%0, %1, %2, %3}, [%4];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x256b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x256b.x2.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b(
  B32 (&out)[8],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b(
  B32 (&out)[8],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x256b.x2.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x256b.x2.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b_pack_16b(
  B32 (&out)[8],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x256b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b_pack_16b(
  B32 (&out)[8],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x256b.x2.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x256b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x256b.x4.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b(
  B32 (&out)[16],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b(
  B32 (&out)[16],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x256b.x4.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x256b.x4.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b_pack_16b(
  B32 (&out)[16],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x256b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b_pack_16b(
  B32 (&out)[16],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x256b.x4.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x256b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x256b.x8.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b(
  B32 (&out)[32],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b(
  B32 (&out)[32],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x256b.x8.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, [%32];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x256b.x8.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b_pack_16b(
  B32 (&out)[32],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x256b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b_pack_16b(
  B32 (&out)[32],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x256b.x8.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, [%32];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x256b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x256b.x16.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b(
  B32 (&out)[64],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b(
  B32 (&out)[64],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x256b.x16.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, [%64];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x256b.x16.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b_pack_16b(
  B32 (&out)[64],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x256b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b_pack_16b(
  B32 (&out)[64],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x256b.x16.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, [%64];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x256b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x256b.x32.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b(
  B32 (&out)[128],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b(
  B32 (&out)[128],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x256b.x32.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, [%128];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63]),
        "=r"(out[64]),
        "=r"(out[65]),
        "=r"(out[66]),
        "=r"(out[67]),
        "=r"(out[68]),
        "=r"(out[69]),
        "=r"(out[70]),
        "=r"(out[71]),
        "=r"(out[72]),
        "=r"(out[73]),
        "=r"(out[74]),
        "=r"(out[75]),
        "=r"(out[76]),
        "=r"(out[77]),
        "=r"(out[78]),
        "=r"(out[79]),
        "=r"(out[80]),
        "=r"(out[81]),
        "=r"(out[82]),
        "=r"(out[83]),
        "=r"(out[84]),
        "=r"(out[85]),
        "=r"(out[86]),
        "=r"(out[87]),
        "=r"(out[88]),
        "=r"(out[89]),
        "=r"(out[90]),
        "=r"(out[91]),
        "=r"(out[92]),
        "=r"(out[93]),
        "=r"(out[94]),
        "=r"(out[95]),
        "=r"(out[96]),
        "=r"(out[97]),
        "=r"(out[98]),
        "=r"(out[99]),
        "=r"(out[100]),
        "=r"(out[101]),
        "=r"(out[102]),
        "=r"(out[103]),
        "=r"(out[104]),
        "=r"(out[105]),
        "=r"(out[106]),
        "=r"(out[107]),
        "=r"(out[108]),
        "=r"(out[109]),
        "=r"(out[110]),
        "=r"(out[111]),
        "=r"(out[112]),
        "=r"(out[113]),
        "=r"(out[114]),
        "=r"(out[115]),
        "=r"(out[116]),
        "=r"(out[117]),
        "=r"(out[118]),
        "=r"(out[119]),
        "=r"(out[120]),
        "=r"(out[121]),
        "=r"(out[122]),
        "=r"(out[123]),
        "=r"(out[124]),
        "=r"(out[125]),
        "=r"(out[126]),
        "=r"(out[127])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x256b.x32.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b_pack_16b(
  B32 (&out)[128],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x256b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_16x256b_pack_16b(
  B32 (&out)[128],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x256b.x32.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, [%128];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63]),
        "=r"(out[64]),
        "=r"(out[65]),
        "=r"(out[66]),
        "=r"(out[67]),
        "=r"(out[68]),
        "=r"(out[69]),
        "=r"(out[70]),
        "=r"(out[71]),
        "=r"(out[72]),
        "=r"(out[73]),
        "=r"(out[74]),
        "=r"(out[75]),
        "=r"(out[76]),
        "=r"(out[77]),
        "=r"(out[78]),
        "=r"(out[79]),
        "=r"(out[80]),
        "=r"(out[81]),
        "=r"(out[82]),
        "=r"(out[83]),
        "=r"(out[84]),
        "=r"(out[85]),
        "=r"(out[86]),
        "=r"(out[87]),
        "=r"(out[88]),
        "=r"(out[89]),
        "=r"(out[90]),
        "=r"(out[91]),
        "=r"(out[92]),
        "=r"(out[93]),
        "=r"(out[94]),
        "=r"(out[95]),
        "=r"(out[96]),
        "=r"(out[97]),
        "=r"(out[98]),
        "=r"(out[99]),
        "=r"(out[100]),
        "=r"(out[101]),
        "=r"(out[102]),
        "=r"(out[103]),
        "=r"(out[104]),
        "=r"(out[105]),
        "=r"(out[106]),
        "=r"(out[107]),
        "=r"(out[108]),
        "=r"(out[109]),
        "=r"(out[110]),
        "=r"(out[111]),
        "=r"(out[112]),
        "=r"(out[113]),
        "=r"(out[114]),
        "=r"(out[115]),
        "=r"(out[116]),
        "=r"(out[117]),
        "=r"(out[118]),
        "=r"(out[119]),
        "=r"(out[120]),
        "=r"(out[121]),
        "=r"(out[122]),
        "=r"(out[123]),
        "=r"(out[124]),
        "=r"(out[125]),
        "=r"(out[126]),
        "=r"(out[127])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x256b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.32x32b.x1.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b(
  B32 (&out)[1],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b(
  B32 (&out)[1],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.32x32b.x1.b32 {%0}, [%1];"
      : "=r"(out[0])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.32x32b.x1.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b_pack_16b(
  B32 (&out)[1],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_32x32b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b_pack_16b(
  B32 (&out)[1],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.32x32b.x1.pack::16b.b32 {%0}, [%1];"
      : "=r"(out[0])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_32x32b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.32x32b.x2.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b(
  B32 (&out)[2],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b(
  B32 (&out)[2],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.32x32b.x2.b32 {%0, %1}, [%2];"
      : "=r"(out[0]),
        "=r"(out[1])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.32x32b.x2.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b_pack_16b(
  B32 (&out)[2],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_32x32b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b_pack_16b(
  B32 (&out)[2],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.32x32b.x2.pack::16b.b32 {%0, %1}, [%2];"
      : "=r"(out[0]),
        "=r"(out[1])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_32x32b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.32x32b.x4.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b(
  B32 (&out)[4],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b(
  B32 (&out)[4],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.32x32b.x4.b32 {%0, %1, %2, %3}, [%4];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.32x32b.x4.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b_pack_16b(
  B32 (&out)[4],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_32x32b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b_pack_16b(
  B32 (&out)[4],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.32x32b.x4.pack::16b.b32 {%0, %1, %2, %3}, [%4];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_32x32b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.32x32b.x8.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b(
  B32 (&out)[8],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b(
  B32 (&out)[8],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.32x32b.x8.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.32x32b.x8.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b_pack_16b(
  B32 (&out)[8],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_32x32b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b_pack_16b(
  B32 (&out)[8],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.32x32b.x8.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_32x32b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.32x32b.x16.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b(
  B32 (&out)[16],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b(
  B32 (&out)[16],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.32x32b.x16.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.32x32b.x16.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b_pack_16b(
  B32 (&out)[16],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_32x32b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b_pack_16b(
  B32 (&out)[16],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.32x32b.x16.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_32x32b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.32x32b.x32.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b(
  B32 (&out)[32],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b(
  B32 (&out)[32],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.32x32b.x32.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, [%32];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.32x32b.x32.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b_pack_16b(
  B32 (&out)[32],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_32x32b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b_pack_16b(
  B32 (&out)[32],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.32x32b.x32.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, [%32];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_32x32b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.32x32b.x64.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b(
  B32 (&out)[64],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b(
  B32 (&out)[64],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.32x32b.x64.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, [%64];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.32x32b.x64.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b_pack_16b(
  B32 (&out)[64],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_32x32b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b_pack_16b(
  B32 (&out)[64],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.32x32b.x64.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, [%64];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_32x32b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.32x32b.x128.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b(
  B32 (&out)[128],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b(
  B32 (&out)[128],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.32x32b.x128.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, [%128];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63]),
        "=r"(out[64]),
        "=r"(out[65]),
        "=r"(out[66]),
        "=r"(out[67]),
        "=r"(out[68]),
        "=r"(out[69]),
        "=r"(out[70]),
        "=r"(out[71]),
        "=r"(out[72]),
        "=r"(out[73]),
        "=r"(out[74]),
        "=r"(out[75]),
        "=r"(out[76]),
        "=r"(out[77]),
        "=r"(out[78]),
        "=r"(out[79]),
        "=r"(out[80]),
        "=r"(out[81]),
        "=r"(out[82]),
        "=r"(out[83]),
        "=r"(out[84]),
        "=r"(out[85]),
        "=r"(out[86]),
        "=r"(out[87]),
        "=r"(out[88]),
        "=r"(out[89]),
        "=r"(out[90]),
        "=r"(out[91]),
        "=r"(out[92]),
        "=r"(out[93]),
        "=r"(out[94]),
        "=r"(out[95]),
        "=r"(out[96]),
        "=r"(out[97]),
        "=r"(out[98]),
        "=r"(out[99]),
        "=r"(out[100]),
        "=r"(out[101]),
        "=r"(out[102]),
        "=r"(out[103]),
        "=r"(out[104]),
        "=r"(out[105]),
        "=r"(out[106]),
        "=r"(out[107]),
        "=r"(out[108]),
        "=r"(out[109]),
        "=r"(out[110]),
        "=r"(out[111]),
        "=r"(out[112]),
        "=r"(out[113]),
        "=r"(out[114]),
        "=r"(out[115]),
        "=r"(out[116]),
        "=r"(out[117]),
        "=r"(out[118]),
        "=r"(out[119]),
        "=r"(out[120]),
        "=r"(out[121]),
        "=r"(out[122]),
        "=r"(out[123]),
        "=r"(out[124]),
        "=r"(out[125]),
        "=r"(out[126]),
        "=r"(out[127])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.32x32b.x128.pack::16b.b32 out, [taddr]; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b_pack_16b(
  B32 (&out)[128],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_32x32b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_ld_32x32b_pack_16b(
  B32 (&out)[128],
  std::uint32_t taddr)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.32x32b.x128.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, [%128];"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63]),
        "=r"(out[64]),
        "=r"(out[65]),
        "=r"(out[66]),
        "=r"(out[67]),
        "=r"(out[68]),
        "=r"(out[69]),
        "=r"(out[70]),
        "=r"(out[71]),
        "=r"(out[72]),
        "=r"(out[73]),
        "=r"(out[74]),
        "=r"(out[75]),
        "=r"(out[76]),
        "=r"(out[77]),
        "=r"(out[78]),
        "=r"(out[79]),
        "=r"(out[80]),
        "=r"(out[81]),
        "=r"(out[82]),
        "=r"(out[83]),
        "=r"(out[84]),
        "=r"(out[85]),
        "=r"(out[86]),
        "=r"(out[87]),
        "=r"(out[88]),
        "=r"(out[89]),
        "=r"(out[90]),
        "=r"(out[91]),
        "=r"(out[92]),
        "=r"(out[93]),
        "=r"(out[94]),
        "=r"(out[95]),
        "=r"(out[96]),
        "=r"(out[97]),
        "=r"(out[98]),
        "=r"(out[99]),
        "=r"(out[100]),
        "=r"(out[101]),
        "=r"(out[102]),
        "=r"(out[103]),
        "=r"(out[104]),
        "=r"(out[105]),
        "=r"(out[106]),
        "=r"(out[107]),
        "=r"(out[108]),
        "=r"(out[109]),
        "=r"(out[110]),
        "=r"(out[111]),
        "=r"(out[112]),
        "=r"(out[113]),
        "=r"(out[114]),
        "=r"(out[115]),
        "=r"(out[116]),
        "=r"(out[117]),
        "=r"(out[118]),
        "=r"(out[119]),
        "=r"(out[120]),
        "=r"(out[121]),
        "=r"(out[122]),
        "=r"(out[123]),
        "=r"(out[124]),
        "=r"(out[125]),
        "=r"(out[126]),
        "=r"(out[127])
      : "r"(taddr)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_32x32b_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x32bx2.x1.b32 out, [taddr], immHalfSplitoff; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2(
  B32 (&out)[1],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2(
  B32 (&out)[1],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x32bx2.x1.b32 {%0}, [%1], %2;"
      : "=r"(out[0])
      : "r"(taddr),
        "n"(immHalfSplitoff.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x32bx2.x1.pack::16b.b32 out, [taddr], immHalfSplitoff; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2_pack_16b(
  B32 (&out)[1],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x32bx2_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2_pack_16b(
  B32 (&out)[1],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x32bx2.x1.pack::16b.b32 {%0}, [%1], %2;"
      : "=r"(out[0])
      : "r"(taddr),
        "n"(immHalfSplitoff.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x32bx2_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x32bx2.x2.b32 out, [taddr], immHalfSplitoff; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2(
  B32 (&out)[2],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2(
  B32 (&out)[2],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x32bx2.x2.b32 {%0, %1}, [%2], %3;"
      : "=r"(out[0]),
        "=r"(out[1])
      : "r"(taddr),
        "n"(immHalfSplitoff.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x32bx2.x2.pack::16b.b32 out, [taddr], immHalfSplitoff; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2_pack_16b(
  B32 (&out)[2],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x32bx2_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2_pack_16b(
  B32 (&out)[2],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x32bx2.x2.pack::16b.b32 {%0, %1}, [%2], %3;"
      : "=r"(out[0]),
        "=r"(out[1])
      : "r"(taddr),
        "n"(immHalfSplitoff.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x32bx2_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x32bx2.x4.b32 out, [taddr], immHalfSplitoff; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2(
  B32 (&out)[4],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2(
  B32 (&out)[4],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x32bx2.x4.b32 {%0, %1, %2, %3}, [%4], %5;"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3])
      : "r"(taddr),
        "n"(immHalfSplitoff.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x32bx2.x4.pack::16b.b32 out, [taddr], immHalfSplitoff; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2_pack_16b(
  B32 (&out)[4],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x32bx2_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2_pack_16b(
  B32 (&out)[4],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x32bx2.x4.pack::16b.b32 {%0, %1, %2, %3}, [%4], %5;"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3])
      : "r"(taddr),
        "n"(immHalfSplitoff.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x32bx2_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x32bx2.x8.b32 out, [taddr], immHalfSplitoff; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2(
  B32 (&out)[8],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2(
  B32 (&out)[8],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x32bx2.x8.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8], %9;"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7])
      : "r"(taddr),
        "n"(immHalfSplitoff.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x32bx2.x8.pack::16b.b32 out, [taddr], immHalfSplitoff; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2_pack_16b(
  B32 (&out)[8],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x32bx2_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2_pack_16b(
  B32 (&out)[8],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x32bx2.x8.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7}, [%8], %9;"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7])
      : "r"(taddr),
        "n"(immHalfSplitoff.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x32bx2_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x32bx2.x16.b32 out, [taddr], immHalfSplitoff; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2(
  B32 (&out)[16],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2(
  B32 (&out)[16],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x32bx2.x16.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16], %17;"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15])
      : "r"(taddr),
        "n"(immHalfSplitoff.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x32bx2.x16.pack::16b.b32 out, [taddr], immHalfSplitoff; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2_pack_16b(
  B32 (&out)[16],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x32bx2_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2_pack_16b(
  B32 (&out)[16],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x32bx2.x16.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, [%16], %17;"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15])
      : "r"(taddr),
        "n"(immHalfSplitoff.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x32bx2_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x32bx2.x32.b32 out, [taddr], immHalfSplitoff; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2(
  B32 (&out)[32],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2(
  B32 (&out)[32],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x32bx2.x32.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, [%32], %33;"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31])
      : "r"(taddr),
        "n"(immHalfSplitoff.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x32bx2.x32.pack::16b.b32 out, [taddr], immHalfSplitoff; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2_pack_16b(
  B32 (&out)[32],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x32bx2_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2_pack_16b(
  B32 (&out)[32],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x32bx2.x32.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, [%32], %33;"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31])
      : "r"(taddr),
        "n"(immHalfSplitoff.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x32bx2_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x32bx2.x64.b32 out, [taddr], immHalfSplitoff; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2(
  B32 (&out)[64],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2(
  B32 (&out)[64],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x32bx2.x64.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, [%64], %65;"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63])
      : "r"(taddr),
        "n"(immHalfSplitoff.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x32bx2.x64.pack::16b.b32 out, [taddr], immHalfSplitoff; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2_pack_16b(
  B32 (&out)[64],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x32bx2_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2_pack_16b(
  B32 (&out)[64],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x32bx2.x64.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, [%64], %65;"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63])
      : "r"(taddr),
        "n"(immHalfSplitoff.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x32bx2_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x32bx2.x128.b32 out, [taddr], immHalfSplitoff; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2(
  B32 (&out)[128],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2(
  B32 (&out)[128],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x32bx2.x128.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, [%128], %129;"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63]),
        "=r"(out[64]),
        "=r"(out[65]),
        "=r"(out[66]),
        "=r"(out[67]),
        "=r"(out[68]),
        "=r"(out[69]),
        "=r"(out[70]),
        "=r"(out[71]),
        "=r"(out[72]),
        "=r"(out[73]),
        "=r"(out[74]),
        "=r"(out[75]),
        "=r"(out[76]),
        "=r"(out[77]),
        "=r"(out[78]),
        "=r"(out[79]),
        "=r"(out[80]),
        "=r"(out[81]),
        "=r"(out[82]),
        "=r"(out[83]),
        "=r"(out[84]),
        "=r"(out[85]),
        "=r"(out[86]),
        "=r"(out[87]),
        "=r"(out[88]),
        "=r"(out[89]),
        "=r"(out[90]),
        "=r"(out[91]),
        "=r"(out[92]),
        "=r"(out[93]),
        "=r"(out[94]),
        "=r"(out[95]),
        "=r"(out[96]),
        "=r"(out[97]),
        "=r"(out[98]),
        "=r"(out[99]),
        "=r"(out[100]),
        "=r"(out[101]),
        "=r"(out[102]),
        "=r"(out[103]),
        "=r"(out[104]),
        "=r"(out[105]),
        "=r"(out[106]),
        "=r"(out[107]),
        "=r"(out[108]),
        "=r"(out[109]),
        "=r"(out[110]),
        "=r"(out[111]),
        "=r"(out[112]),
        "=r"(out[113]),
        "=r"(out[114]),
        "=r"(out[115]),
        "=r"(out[116]),
        "=r"(out[117]),
        "=r"(out[118]),
        "=r"(out[119]),
        "=r"(out[120]),
        "=r"(out[121]),
        "=r"(out[122]),
        "=r"(out[123]),
        "=r"(out[124]),
        "=r"(out[125]),
        "=r"(out[126]),
        "=r"(out[127])
      : "r"(taddr),
        "n"(immHalfSplitoff.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.sync.aligned.16x32bx2.x128.pack::16b.b32 out, [taddr], immHalfSplitoff; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2_pack_16b(
  B32 (&out)[128],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_ld_16x32bx2_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true, int N32>
__device__ static inline void tcgen05_ld_16x32bx2_pack_16b(
  B32 (&out)[128],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.ld.sync.aligned.16x32bx2.x128.pack::16b.b32 {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, [%128], %129;"
      : "=r"(out[0]),
        "=r"(out[1]),
        "=r"(out[2]),
        "=r"(out[3]),
        "=r"(out[4]),
        "=r"(out[5]),
        "=r"(out[6]),
        "=r"(out[7]),
        "=r"(out[8]),
        "=r"(out[9]),
        "=r"(out[10]),
        "=r"(out[11]),
        "=r"(out[12]),
        "=r"(out[13]),
        "=r"(out[14]),
        "=r"(out[15]),
        "=r"(out[16]),
        "=r"(out[17]),
        "=r"(out[18]),
        "=r"(out[19]),
        "=r"(out[20]),
        "=r"(out[21]),
        "=r"(out[22]),
        "=r"(out[23]),
        "=r"(out[24]),
        "=r"(out[25]),
        "=r"(out[26]),
        "=r"(out[27]),
        "=r"(out[28]),
        "=r"(out[29]),
        "=r"(out[30]),
        "=r"(out[31]),
        "=r"(out[32]),
        "=r"(out[33]),
        "=r"(out[34]),
        "=r"(out[35]),
        "=r"(out[36]),
        "=r"(out[37]),
        "=r"(out[38]),
        "=r"(out[39]),
        "=r"(out[40]),
        "=r"(out[41]),
        "=r"(out[42]),
        "=r"(out[43]),
        "=r"(out[44]),
        "=r"(out[45]),
        "=r"(out[46]),
        "=r"(out[47]),
        "=r"(out[48]),
        "=r"(out[49]),
        "=r"(out[50]),
        "=r"(out[51]),
        "=r"(out[52]),
        "=r"(out[53]),
        "=r"(out[54]),
        "=r"(out[55]),
        "=r"(out[56]),
        "=r"(out[57]),
        "=r"(out[58]),
        "=r"(out[59]),
        "=r"(out[60]),
        "=r"(out[61]),
        "=r"(out[62]),
        "=r"(out[63]),
        "=r"(out[64]),
        "=r"(out[65]),
        "=r"(out[66]),
        "=r"(out[67]),
        "=r"(out[68]),
        "=r"(out[69]),
        "=r"(out[70]),
        "=r"(out[71]),
        "=r"(out[72]),
        "=r"(out[73]),
        "=r"(out[74]),
        "=r"(out[75]),
        "=r"(out[76]),
        "=r"(out[77]),
        "=r"(out[78]),
        "=r"(out[79]),
        "=r"(out[80]),
        "=r"(out[81]),
        "=r"(out[82]),
        "=r"(out[83]),
        "=r"(out[84]),
        "=r"(out[85]),
        "=r"(out[86]),
        "=r"(out[87]),
        "=r"(out[88]),
        "=r"(out[89]),
        "=r"(out[90]),
        "=r"(out[91]),
        "=r"(out[92]),
        "=r"(out[93]),
        "=r"(out[94]),
        "=r"(out[95]),
        "=r"(out[96]),
        "=r"(out[97]),
        "=r"(out[98]),
        "=r"(out[99]),
        "=r"(out[100]),
        "=r"(out[101]),
        "=r"(out[102]),
        "=r"(out[103]),
        "=r"(out[104]),
        "=r"(out[105]),
        "=r"(out[106]),
        "=r"(out[107]),
        "=r"(out[108]),
        "=r"(out[109]),
        "=r"(out[110]),
        "=r"(out[111]),
        "=r"(out[112]),
        "=r"(out[113]),
        "=r"(out[114]),
        "=r"(out[115]),
        "=r"(out[116]),
        "=r"(out[117]),
        "=r"(out[118]),
        "=r"(out[119]),
        "=r"(out[120]),
        "=r"(out[121]),
        "=r"(out[122]),
        "=r"(out[123]),
        "=r"(out[124]),
        "=r"(out[125]),
        "=r"(out[126]),
        "=r"(out[127])
      : "r"(taddr),
        "n"(immHalfSplitoff.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_16x32bx2_pack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.ld.red.sync.aligned.32x32b.x2.u32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[2],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[2],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::uint32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x2.u32.min {%0, %1}, %2, [%3];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x2.u32.max {%0, %1}, %2, [%3];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x2.s32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[2],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[2],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::int32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x2.s32.min {%0, %1}, %2, [%3];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x2.s32.max {%0, %1}, %2, [%3];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x2.f32.op.abs out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[2],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[2],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x2.f32.min.abs {%0, %1}, %2, [%3];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x2.f32.max.abs {%0, %1}, %2, [%3];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x2.f32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  float (&out)[2],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  float (&out)[2],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x2.f32.min {%0, %1}, %2, [%3];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x2.f32.max {%0, %1}, %2, [%3];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x4.u32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[4],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[4],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::uint32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x4.u32.min {%0, %1, %2, %3}, %4, [%5];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x4.u32.max {%0, %1, %2, %3}, %4, [%5];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x4.s32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[4],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[4],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::int32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x4.s32.min {%0, %1, %2, %3}, %4, [%5];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x4.s32.max {%0, %1, %2, %3}, %4, [%5];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x4.f32.op.abs out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[4],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[4],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x4.f32.min.abs {%0, %1, %2, %3}, %4, [%5];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x4.f32.max.abs {%0, %1, %2, %3}, %4, [%5];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x4.f32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  float (&out)[4],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  float (&out)[4],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x4.f32.min {%0, %1, %2, %3}, %4, [%5];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x4.f32.max {%0, %1, %2, %3}, %4, [%5];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x8.u32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[8],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[8],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::uint32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x8.u32.min {%0, %1, %2, %3, %4, %5, %6, %7}, %8, [%9];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x8.u32.max {%0, %1, %2, %3, %4, %5, %6, %7}, %8, [%9];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x8.s32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[8],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[8],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::int32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x8.s32.min {%0, %1, %2, %3, %4, %5, %6, %7}, %8, [%9];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x8.s32.max {%0, %1, %2, %3, %4, %5, %6, %7}, %8, [%9];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x8.f32.op.abs out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[8],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[8],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x8.f32.min.abs {%0, %1, %2, %3, %4, %5, %6, %7}, %8, [%9];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x8.f32.max.abs {%0, %1, %2, %3, %4, %5, %6, %7}, %8, [%9];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x8.f32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  float (&out)[8],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  float (&out)[8],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x8.f32.min {%0, %1, %2, %3, %4, %5, %6, %7}, %8, [%9];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x8.f32.max {%0, %1, %2, %3, %4, %5, %6, %7}, %8, [%9];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x16.u32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[16],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[16],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::uint32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x16.u32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, %16, [%17];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x16.u32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, %16, [%17];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x16.s32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[16],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[16],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::int32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x16.s32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, %16, [%17];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x16.s32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, %16, [%17];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x16.f32.op.abs out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[16],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[16],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x16.f32.min.abs {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, %16, [%17];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x16.f32.max.abs {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, %16, [%17];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x16.f32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  float (&out)[16],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  float (&out)[16],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x16.f32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, %16, [%17];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x16.f32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, %16, [%17];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x32.u32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[32],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[32],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::uint32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x32.u32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, %32, [%33];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x32.u32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, %32, [%33];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x32.s32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[32],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[32],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::int32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x32.s32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, %32, [%33];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x32.s32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, %32, [%33];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x32.f32.op.abs out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[32],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[32],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x32.f32.min.abs {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, %32, [%33];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x32.f32.max.abs {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, %32, [%33];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x32.f32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  float (&out)[32],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  float (&out)[32],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x32.f32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, %32, [%33];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x32.f32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, %32, [%33];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x64.u32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[64],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[64],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::uint32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x64.u32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, %64, [%65];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(out[32]),
          "=r"(out[33]),
          "=r"(out[34]),
          "=r"(out[35]),
          "=r"(out[36]),
          "=r"(out[37]),
          "=r"(out[38]),
          "=r"(out[39]),
          "=r"(out[40]),
          "=r"(out[41]),
          "=r"(out[42]),
          "=r"(out[43]),
          "=r"(out[44]),
          "=r"(out[45]),
          "=r"(out[46]),
          "=r"(out[47]),
          "=r"(out[48]),
          "=r"(out[49]),
          "=r"(out[50]),
          "=r"(out[51]),
          "=r"(out[52]),
          "=r"(out[53]),
          "=r"(out[54]),
          "=r"(out[55]),
          "=r"(out[56]),
          "=r"(out[57]),
          "=r"(out[58]),
          "=r"(out[59]),
          "=r"(out[60]),
          "=r"(out[61]),
          "=r"(out[62]),
          "=r"(out[63]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x64.u32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, %64, [%65];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(out[32]),
          "=r"(out[33]),
          "=r"(out[34]),
          "=r"(out[35]),
          "=r"(out[36]),
          "=r"(out[37]),
          "=r"(out[38]),
          "=r"(out[39]),
          "=r"(out[40]),
          "=r"(out[41]),
          "=r"(out[42]),
          "=r"(out[43]),
          "=r"(out[44]),
          "=r"(out[45]),
          "=r"(out[46]),
          "=r"(out[47]),
          "=r"(out[48]),
          "=r"(out[49]),
          "=r"(out[50]),
          "=r"(out[51]),
          "=r"(out[52]),
          "=r"(out[53]),
          "=r"(out[54]),
          "=r"(out[55]),
          "=r"(out[56]),
          "=r"(out[57]),
          "=r"(out[58]),
          "=r"(out[59]),
          "=r"(out[60]),
          "=r"(out[61]),
          "=r"(out[62]),
          "=r"(out[63]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x64.s32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[64],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[64],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::int32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x64.s32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, %64, [%65];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(out[32]),
          "=r"(out[33]),
          "=r"(out[34]),
          "=r"(out[35]),
          "=r"(out[36]),
          "=r"(out[37]),
          "=r"(out[38]),
          "=r"(out[39]),
          "=r"(out[40]),
          "=r"(out[41]),
          "=r"(out[42]),
          "=r"(out[43]),
          "=r"(out[44]),
          "=r"(out[45]),
          "=r"(out[46]),
          "=r"(out[47]),
          "=r"(out[48]),
          "=r"(out[49]),
          "=r"(out[50]),
          "=r"(out[51]),
          "=r"(out[52]),
          "=r"(out[53]),
          "=r"(out[54]),
          "=r"(out[55]),
          "=r"(out[56]),
          "=r"(out[57]),
          "=r"(out[58]),
          "=r"(out[59]),
          "=r"(out[60]),
          "=r"(out[61]),
          "=r"(out[62]),
          "=r"(out[63]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x64.s32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, %64, [%65];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(out[32]),
          "=r"(out[33]),
          "=r"(out[34]),
          "=r"(out[35]),
          "=r"(out[36]),
          "=r"(out[37]),
          "=r"(out[38]),
          "=r"(out[39]),
          "=r"(out[40]),
          "=r"(out[41]),
          "=r"(out[42]),
          "=r"(out[43]),
          "=r"(out[44]),
          "=r"(out[45]),
          "=r"(out[46]),
          "=r"(out[47]),
          "=r"(out[48]),
          "=r"(out[49]),
          "=r"(out[50]),
          "=r"(out[51]),
          "=r"(out[52]),
          "=r"(out[53]),
          "=r"(out[54]),
          "=r"(out[55]),
          "=r"(out[56]),
          "=r"(out[57]),
          "=r"(out[58]),
          "=r"(out[59]),
          "=r"(out[60]),
          "=r"(out[61]),
          "=r"(out[62]),
          "=r"(out[63]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x64.f32.op.abs out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[64],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[64],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x64.f32.min.abs {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, %64, [%65];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(out[32]),
          "=f"(out[33]),
          "=f"(out[34]),
          "=f"(out[35]),
          "=f"(out[36]),
          "=f"(out[37]),
          "=f"(out[38]),
          "=f"(out[39]),
          "=f"(out[40]),
          "=f"(out[41]),
          "=f"(out[42]),
          "=f"(out[43]),
          "=f"(out[44]),
          "=f"(out[45]),
          "=f"(out[46]),
          "=f"(out[47]),
          "=f"(out[48]),
          "=f"(out[49]),
          "=f"(out[50]),
          "=f"(out[51]),
          "=f"(out[52]),
          "=f"(out[53]),
          "=f"(out[54]),
          "=f"(out[55]),
          "=f"(out[56]),
          "=f"(out[57]),
          "=f"(out[58]),
          "=f"(out[59]),
          "=f"(out[60]),
          "=f"(out[61]),
          "=f"(out[62]),
          "=f"(out[63]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x64.f32.max.abs {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, %64, [%65];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(out[32]),
          "=f"(out[33]),
          "=f"(out[34]),
          "=f"(out[35]),
          "=f"(out[36]),
          "=f"(out[37]),
          "=f"(out[38]),
          "=f"(out[39]),
          "=f"(out[40]),
          "=f"(out[41]),
          "=f"(out[42]),
          "=f"(out[43]),
          "=f"(out[44]),
          "=f"(out[45]),
          "=f"(out[46]),
          "=f"(out[47]),
          "=f"(out[48]),
          "=f"(out[49]),
          "=f"(out[50]),
          "=f"(out[51]),
          "=f"(out[52]),
          "=f"(out[53]),
          "=f"(out[54]),
          "=f"(out[55]),
          "=f"(out[56]),
          "=f"(out[57]),
          "=f"(out[58]),
          "=f"(out[59]),
          "=f"(out[60]),
          "=f"(out[61]),
          "=f"(out[62]),
          "=f"(out[63]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x64.f32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  float (&out)[64],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  float (&out)[64],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x64.f32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, %64, [%65];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(out[32]),
          "=f"(out[33]),
          "=f"(out[34]),
          "=f"(out[35]),
          "=f"(out[36]),
          "=f"(out[37]),
          "=f"(out[38]),
          "=f"(out[39]),
          "=f"(out[40]),
          "=f"(out[41]),
          "=f"(out[42]),
          "=f"(out[43]),
          "=f"(out[44]),
          "=f"(out[45]),
          "=f"(out[46]),
          "=f"(out[47]),
          "=f"(out[48]),
          "=f"(out[49]),
          "=f"(out[50]),
          "=f"(out[51]),
          "=f"(out[52]),
          "=f"(out[53]),
          "=f"(out[54]),
          "=f"(out[55]),
          "=f"(out[56]),
          "=f"(out[57]),
          "=f"(out[58]),
          "=f"(out[59]),
          "=f"(out[60]),
          "=f"(out[61]),
          "=f"(out[62]),
          "=f"(out[63]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x64.f32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, %64, [%65];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(out[32]),
          "=f"(out[33]),
          "=f"(out[34]),
          "=f"(out[35]),
          "=f"(out[36]),
          "=f"(out[37]),
          "=f"(out[38]),
          "=f"(out[39]),
          "=f"(out[40]),
          "=f"(out[41]),
          "=f"(out[42]),
          "=f"(out[43]),
          "=f"(out[44]),
          "=f"(out[45]),
          "=f"(out[46]),
          "=f"(out[47]),
          "=f"(out[48]),
          "=f"(out[49]),
          "=f"(out[50]),
          "=f"(out[51]),
          "=f"(out[52]),
          "=f"(out[53]),
          "=f"(out[54]),
          "=f"(out[55]),
          "=f"(out[56]),
          "=f"(out[57]),
          "=f"(out[58]),
          "=f"(out[59]),
          "=f"(out[60]),
          "=f"(out[61]),
          "=f"(out[62]),
          "=f"(out[63]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x128.u32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[128],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[128],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::uint32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x128.u32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, %128, [%129];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(out[32]),
          "=r"(out[33]),
          "=r"(out[34]),
          "=r"(out[35]),
          "=r"(out[36]),
          "=r"(out[37]),
          "=r"(out[38]),
          "=r"(out[39]),
          "=r"(out[40]),
          "=r"(out[41]),
          "=r"(out[42]),
          "=r"(out[43]),
          "=r"(out[44]),
          "=r"(out[45]),
          "=r"(out[46]),
          "=r"(out[47]),
          "=r"(out[48]),
          "=r"(out[49]),
          "=r"(out[50]),
          "=r"(out[51]),
          "=r"(out[52]),
          "=r"(out[53]),
          "=r"(out[54]),
          "=r"(out[55]),
          "=r"(out[56]),
          "=r"(out[57]),
          "=r"(out[58]),
          "=r"(out[59]),
          "=r"(out[60]),
          "=r"(out[61]),
          "=r"(out[62]),
          "=r"(out[63]),
          "=r"(out[64]),
          "=r"(out[65]),
          "=r"(out[66]),
          "=r"(out[67]),
          "=r"(out[68]),
          "=r"(out[69]),
          "=r"(out[70]),
          "=r"(out[71]),
          "=r"(out[72]),
          "=r"(out[73]),
          "=r"(out[74]),
          "=r"(out[75]),
          "=r"(out[76]),
          "=r"(out[77]),
          "=r"(out[78]),
          "=r"(out[79]),
          "=r"(out[80]),
          "=r"(out[81]),
          "=r"(out[82]),
          "=r"(out[83]),
          "=r"(out[84]),
          "=r"(out[85]),
          "=r"(out[86]),
          "=r"(out[87]),
          "=r"(out[88]),
          "=r"(out[89]),
          "=r"(out[90]),
          "=r"(out[91]),
          "=r"(out[92]),
          "=r"(out[93]),
          "=r"(out[94]),
          "=r"(out[95]),
          "=r"(out[96]),
          "=r"(out[97]),
          "=r"(out[98]),
          "=r"(out[99]),
          "=r"(out[100]),
          "=r"(out[101]),
          "=r"(out[102]),
          "=r"(out[103]),
          "=r"(out[104]),
          "=r"(out[105]),
          "=r"(out[106]),
          "=r"(out[107]),
          "=r"(out[108]),
          "=r"(out[109]),
          "=r"(out[110]),
          "=r"(out[111]),
          "=r"(out[112]),
          "=r"(out[113]),
          "=r"(out[114]),
          "=r"(out[115]),
          "=r"(out[116]),
          "=r"(out[117]),
          "=r"(out[118]),
          "=r"(out[119]),
          "=r"(out[120]),
          "=r"(out[121]),
          "=r"(out[122]),
          "=r"(out[123]),
          "=r"(out[124]),
          "=r"(out[125]),
          "=r"(out[126]),
          "=r"(out[127]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x128.u32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, %128, [%129];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(out[32]),
          "=r"(out[33]),
          "=r"(out[34]),
          "=r"(out[35]),
          "=r"(out[36]),
          "=r"(out[37]),
          "=r"(out[38]),
          "=r"(out[39]),
          "=r"(out[40]),
          "=r"(out[41]),
          "=r"(out[42]),
          "=r"(out[43]),
          "=r"(out[44]),
          "=r"(out[45]),
          "=r"(out[46]),
          "=r"(out[47]),
          "=r"(out[48]),
          "=r"(out[49]),
          "=r"(out[50]),
          "=r"(out[51]),
          "=r"(out[52]),
          "=r"(out[53]),
          "=r"(out[54]),
          "=r"(out[55]),
          "=r"(out[56]),
          "=r"(out[57]),
          "=r"(out[58]),
          "=r"(out[59]),
          "=r"(out[60]),
          "=r"(out[61]),
          "=r"(out[62]),
          "=r"(out[63]),
          "=r"(out[64]),
          "=r"(out[65]),
          "=r"(out[66]),
          "=r"(out[67]),
          "=r"(out[68]),
          "=r"(out[69]),
          "=r"(out[70]),
          "=r"(out[71]),
          "=r"(out[72]),
          "=r"(out[73]),
          "=r"(out[74]),
          "=r"(out[75]),
          "=r"(out[76]),
          "=r"(out[77]),
          "=r"(out[78]),
          "=r"(out[79]),
          "=r"(out[80]),
          "=r"(out[81]),
          "=r"(out[82]),
          "=r"(out[83]),
          "=r"(out[84]),
          "=r"(out[85]),
          "=r"(out[86]),
          "=r"(out[87]),
          "=r"(out[88]),
          "=r"(out[89]),
          "=r"(out[90]),
          "=r"(out[91]),
          "=r"(out[92]),
          "=r"(out[93]),
          "=r"(out[94]),
          "=r"(out[95]),
          "=r"(out[96]),
          "=r"(out[97]),
          "=r"(out[98]),
          "=r"(out[99]),
          "=r"(out[100]),
          "=r"(out[101]),
          "=r"(out[102]),
          "=r"(out[103]),
          "=r"(out[104]),
          "=r"(out[105]),
          "=r"(out[106]),
          "=r"(out[107]),
          "=r"(out[108]),
          "=r"(out[109]),
          "=r"(out[110]),
          "=r"(out[111]),
          "=r"(out[112]),
          "=r"(out[113]),
          "=r"(out[114]),
          "=r"(out[115]),
          "=r"(out[116]),
          "=r"(out[117]),
          "=r"(out[118]),
          "=r"(out[119]),
          "=r"(out[120]),
          "=r"(out[121]),
          "=r"(out[122]),
          "=r"(out[123]),
          "=r"(out[124]),
          "=r"(out[125]),
          "=r"(out[126]),
          "=r"(out[127]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x128.s32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[128],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[128],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::int32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x128.s32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, %128, [%129];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(out[32]),
          "=r"(out[33]),
          "=r"(out[34]),
          "=r"(out[35]),
          "=r"(out[36]),
          "=r"(out[37]),
          "=r"(out[38]),
          "=r"(out[39]),
          "=r"(out[40]),
          "=r"(out[41]),
          "=r"(out[42]),
          "=r"(out[43]),
          "=r"(out[44]),
          "=r"(out[45]),
          "=r"(out[46]),
          "=r"(out[47]),
          "=r"(out[48]),
          "=r"(out[49]),
          "=r"(out[50]),
          "=r"(out[51]),
          "=r"(out[52]),
          "=r"(out[53]),
          "=r"(out[54]),
          "=r"(out[55]),
          "=r"(out[56]),
          "=r"(out[57]),
          "=r"(out[58]),
          "=r"(out[59]),
          "=r"(out[60]),
          "=r"(out[61]),
          "=r"(out[62]),
          "=r"(out[63]),
          "=r"(out[64]),
          "=r"(out[65]),
          "=r"(out[66]),
          "=r"(out[67]),
          "=r"(out[68]),
          "=r"(out[69]),
          "=r"(out[70]),
          "=r"(out[71]),
          "=r"(out[72]),
          "=r"(out[73]),
          "=r"(out[74]),
          "=r"(out[75]),
          "=r"(out[76]),
          "=r"(out[77]),
          "=r"(out[78]),
          "=r"(out[79]),
          "=r"(out[80]),
          "=r"(out[81]),
          "=r"(out[82]),
          "=r"(out[83]),
          "=r"(out[84]),
          "=r"(out[85]),
          "=r"(out[86]),
          "=r"(out[87]),
          "=r"(out[88]),
          "=r"(out[89]),
          "=r"(out[90]),
          "=r"(out[91]),
          "=r"(out[92]),
          "=r"(out[93]),
          "=r"(out[94]),
          "=r"(out[95]),
          "=r"(out[96]),
          "=r"(out[97]),
          "=r"(out[98]),
          "=r"(out[99]),
          "=r"(out[100]),
          "=r"(out[101]),
          "=r"(out[102]),
          "=r"(out[103]),
          "=r"(out[104]),
          "=r"(out[105]),
          "=r"(out[106]),
          "=r"(out[107]),
          "=r"(out[108]),
          "=r"(out[109]),
          "=r"(out[110]),
          "=r"(out[111]),
          "=r"(out[112]),
          "=r"(out[113]),
          "=r"(out[114]),
          "=r"(out[115]),
          "=r"(out[116]),
          "=r"(out[117]),
          "=r"(out[118]),
          "=r"(out[119]),
          "=r"(out[120]),
          "=r"(out[121]),
          "=r"(out[122]),
          "=r"(out[123]),
          "=r"(out[124]),
          "=r"(out[125]),
          "=r"(out[126]),
          "=r"(out[127]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x128.s32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, %128, [%129];"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(out[32]),
          "=r"(out[33]),
          "=r"(out[34]),
          "=r"(out[35]),
          "=r"(out[36]),
          "=r"(out[37]),
          "=r"(out[38]),
          "=r"(out[39]),
          "=r"(out[40]),
          "=r"(out[41]),
          "=r"(out[42]),
          "=r"(out[43]),
          "=r"(out[44]),
          "=r"(out[45]),
          "=r"(out[46]),
          "=r"(out[47]),
          "=r"(out[48]),
          "=r"(out[49]),
          "=r"(out[50]),
          "=r"(out[51]),
          "=r"(out[52]),
          "=r"(out[53]),
          "=r"(out[54]),
          "=r"(out[55]),
          "=r"(out[56]),
          "=r"(out[57]),
          "=r"(out[58]),
          "=r"(out[59]),
          "=r"(out[60]),
          "=r"(out[61]),
          "=r"(out[62]),
          "=r"(out[63]),
          "=r"(out[64]),
          "=r"(out[65]),
          "=r"(out[66]),
          "=r"(out[67]),
          "=r"(out[68]),
          "=r"(out[69]),
          "=r"(out[70]),
          "=r"(out[71]),
          "=r"(out[72]),
          "=r"(out[73]),
          "=r"(out[74]),
          "=r"(out[75]),
          "=r"(out[76]),
          "=r"(out[77]),
          "=r"(out[78]),
          "=r"(out[79]),
          "=r"(out[80]),
          "=r"(out[81]),
          "=r"(out[82]),
          "=r"(out[83]),
          "=r"(out[84]),
          "=r"(out[85]),
          "=r"(out[86]),
          "=r"(out[87]),
          "=r"(out[88]),
          "=r"(out[89]),
          "=r"(out[90]),
          "=r"(out[91]),
          "=r"(out[92]),
          "=r"(out[93]),
          "=r"(out[94]),
          "=r"(out[95]),
          "=r"(out[96]),
          "=r"(out[97]),
          "=r"(out[98]),
          "=r"(out[99]),
          "=r"(out[100]),
          "=r"(out[101]),
          "=r"(out[102]),
          "=r"(out[103]),
          "=r"(out[104]),
          "=r"(out[105]),
          "=r"(out[106]),
          "=r"(out[107]),
          "=r"(out[108]),
          "=r"(out[109]),
          "=r"(out[110]),
          "=r"(out[111]),
          "=r"(out[112]),
          "=r"(out[113]),
          "=r"(out[114]),
          "=r"(out[115]),
          "=r"(out[116]),
          "=r"(out[117]),
          "=r"(out[118]),
          "=r"(out[119]),
          "=r"(out[120]),
          "=r"(out[121]),
          "=r"(out[122]),
          "=r"(out[123]),
          "=r"(out[124]),
          "=r"(out[125]),
          "=r"(out[126]),
          "=r"(out[127]),
          "=r"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x128.f32.op.abs out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[128],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[128],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x128.f32.min.abs {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, %128, [%129];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(out[32]),
          "=f"(out[33]),
          "=f"(out[34]),
          "=f"(out[35]),
          "=f"(out[36]),
          "=f"(out[37]),
          "=f"(out[38]),
          "=f"(out[39]),
          "=f"(out[40]),
          "=f"(out[41]),
          "=f"(out[42]),
          "=f"(out[43]),
          "=f"(out[44]),
          "=f"(out[45]),
          "=f"(out[46]),
          "=f"(out[47]),
          "=f"(out[48]),
          "=f"(out[49]),
          "=f"(out[50]),
          "=f"(out[51]),
          "=f"(out[52]),
          "=f"(out[53]),
          "=f"(out[54]),
          "=f"(out[55]),
          "=f"(out[56]),
          "=f"(out[57]),
          "=f"(out[58]),
          "=f"(out[59]),
          "=f"(out[60]),
          "=f"(out[61]),
          "=f"(out[62]),
          "=f"(out[63]),
          "=f"(out[64]),
          "=f"(out[65]),
          "=f"(out[66]),
          "=f"(out[67]),
          "=f"(out[68]),
          "=f"(out[69]),
          "=f"(out[70]),
          "=f"(out[71]),
          "=f"(out[72]),
          "=f"(out[73]),
          "=f"(out[74]),
          "=f"(out[75]),
          "=f"(out[76]),
          "=f"(out[77]),
          "=f"(out[78]),
          "=f"(out[79]),
          "=f"(out[80]),
          "=f"(out[81]),
          "=f"(out[82]),
          "=f"(out[83]),
          "=f"(out[84]),
          "=f"(out[85]),
          "=f"(out[86]),
          "=f"(out[87]),
          "=f"(out[88]),
          "=f"(out[89]),
          "=f"(out[90]),
          "=f"(out[91]),
          "=f"(out[92]),
          "=f"(out[93]),
          "=f"(out[94]),
          "=f"(out[95]),
          "=f"(out[96]),
          "=f"(out[97]),
          "=f"(out[98]),
          "=f"(out[99]),
          "=f"(out[100]),
          "=f"(out[101]),
          "=f"(out[102]),
          "=f"(out[103]),
          "=f"(out[104]),
          "=f"(out[105]),
          "=f"(out[106]),
          "=f"(out[107]),
          "=f"(out[108]),
          "=f"(out[109]),
          "=f"(out[110]),
          "=f"(out[111]),
          "=f"(out[112]),
          "=f"(out[113]),
          "=f"(out[114]),
          "=f"(out[115]),
          "=f"(out[116]),
          "=f"(out[117]),
          "=f"(out[118]),
          "=f"(out[119]),
          "=f"(out[120]),
          "=f"(out[121]),
          "=f"(out[122]),
          "=f"(out[123]),
          "=f"(out[124]),
          "=f"(out[125]),
          "=f"(out[126]),
          "=f"(out[127]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x128.f32.max.abs {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, %128, [%129];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(out[32]),
          "=f"(out[33]),
          "=f"(out[34]),
          "=f"(out[35]),
          "=f"(out[36]),
          "=f"(out[37]),
          "=f"(out[38]),
          "=f"(out[39]),
          "=f"(out[40]),
          "=f"(out[41]),
          "=f"(out[42]),
          "=f"(out[43]),
          "=f"(out[44]),
          "=f"(out[45]),
          "=f"(out[46]),
          "=f"(out[47]),
          "=f"(out[48]),
          "=f"(out[49]),
          "=f"(out[50]),
          "=f"(out[51]),
          "=f"(out[52]),
          "=f"(out[53]),
          "=f"(out[54]),
          "=f"(out[55]),
          "=f"(out[56]),
          "=f"(out[57]),
          "=f"(out[58]),
          "=f"(out[59]),
          "=f"(out[60]),
          "=f"(out[61]),
          "=f"(out[62]),
          "=f"(out[63]),
          "=f"(out[64]),
          "=f"(out[65]),
          "=f"(out[66]),
          "=f"(out[67]),
          "=f"(out[68]),
          "=f"(out[69]),
          "=f"(out[70]),
          "=f"(out[71]),
          "=f"(out[72]),
          "=f"(out[73]),
          "=f"(out[74]),
          "=f"(out[75]),
          "=f"(out[76]),
          "=f"(out[77]),
          "=f"(out[78]),
          "=f"(out[79]),
          "=f"(out[80]),
          "=f"(out[81]),
          "=f"(out[82]),
          "=f"(out[83]),
          "=f"(out[84]),
          "=f"(out[85]),
          "=f"(out[86]),
          "=f"(out[87]),
          "=f"(out[88]),
          "=f"(out[89]),
          "=f"(out[90]),
          "=f"(out[91]),
          "=f"(out[92]),
          "=f"(out[93]),
          "=f"(out[94]),
          "=f"(out[95]),
          "=f"(out[96]),
          "=f"(out[97]),
          "=f"(out[98]),
          "=f"(out[99]),
          "=f"(out[100]),
          "=f"(out[101]),
          "=f"(out[102]),
          "=f"(out[103]),
          "=f"(out[104]),
          "=f"(out[105]),
          "=f"(out[106]),
          "=f"(out[107]),
          "=f"(out[108]),
          "=f"(out[109]),
          "=f"(out[110]),
          "=f"(out[111]),
          "=f"(out[112]),
          "=f"(out[113]),
          "=f"(out[114]),
          "=f"(out[115]),
          "=f"(out[116]),
          "=f"(out[117]),
          "=f"(out[118]),
          "=f"(out[119]),
          "=f"(out[120]),
          "=f"(out[121]),
          "=f"(out[122]),
          "=f"(out[123]),
          "=f"(out[124]),
          "=f"(out[125]),
          "=f"(out[126]),
          "=f"(out[127]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.32x32b.x128.f32.op out, redval, [taddr]; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  float (&out)[128],
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_32x32b(
  cuda_ptx::op_t<Op> op,
  float (&out)[128],
  std::uint32_t taddr)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x128.f32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, %128, [%129];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(out[32]),
          "=f"(out[33]),
          "=f"(out[34]),
          "=f"(out[35]),
          "=f"(out[36]),
          "=f"(out[37]),
          "=f"(out[38]),
          "=f"(out[39]),
          "=f"(out[40]),
          "=f"(out[41]),
          "=f"(out[42]),
          "=f"(out[43]),
          "=f"(out[44]),
          "=f"(out[45]),
          "=f"(out[46]),
          "=f"(out[47]),
          "=f"(out[48]),
          "=f"(out[49]),
          "=f"(out[50]),
          "=f"(out[51]),
          "=f"(out[52]),
          "=f"(out[53]),
          "=f"(out[54]),
          "=f"(out[55]),
          "=f"(out[56]),
          "=f"(out[57]),
          "=f"(out[58]),
          "=f"(out[59]),
          "=f"(out[60]),
          "=f"(out[61]),
          "=f"(out[62]),
          "=f"(out[63]),
          "=f"(out[64]),
          "=f"(out[65]),
          "=f"(out[66]),
          "=f"(out[67]),
          "=f"(out[68]),
          "=f"(out[69]),
          "=f"(out[70]),
          "=f"(out[71]),
          "=f"(out[72]),
          "=f"(out[73]),
          "=f"(out[74]),
          "=f"(out[75]),
          "=f"(out[76]),
          "=f"(out[77]),
          "=f"(out[78]),
          "=f"(out[79]),
          "=f"(out[80]),
          "=f"(out[81]),
          "=f"(out[82]),
          "=f"(out[83]),
          "=f"(out[84]),
          "=f"(out[85]),
          "=f"(out[86]),
          "=f"(out[87]),
          "=f"(out[88]),
          "=f"(out[89]),
          "=f"(out[90]),
          "=f"(out[91]),
          "=f"(out[92]),
          "=f"(out[93]),
          "=f"(out[94]),
          "=f"(out[95]),
          "=f"(out[96]),
          "=f"(out[97]),
          "=f"(out[98]),
          "=f"(out[99]),
          "=f"(out[100]),
          "=f"(out[101]),
          "=f"(out[102]),
          "=f"(out[103]),
          "=f"(out[104]),
          "=f"(out[105]),
          "=f"(out[106]),
          "=f"(out[107]),
          "=f"(out[108]),
          "=f"(out[109]),
          "=f"(out[110]),
          "=f"(out[111]),
          "=f"(out[112]),
          "=f"(out[113]),
          "=f"(out[114]),
          "=f"(out[115]),
          "=f"(out[116]),
          "=f"(out[117]),
          "=f"(out[118]),
          "=f"(out[119]),
          "=f"(out[120]),
          "=f"(out[121]),
          "=f"(out[122]),
          "=f"(out[123]),
          "=f"(out[124]),
          "=f"(out[125]),
          "=f"(out[126]),
          "=f"(out[127]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.32x32b.x128.f32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, %128, [%129];"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(out[32]),
          "=f"(out[33]),
          "=f"(out[34]),
          "=f"(out[35]),
          "=f"(out[36]),
          "=f"(out[37]),
          "=f"(out[38]),
          "=f"(out[39]),
          "=f"(out[40]),
          "=f"(out[41]),
          "=f"(out[42]),
          "=f"(out[43]),
          "=f"(out[44]),
          "=f"(out[45]),
          "=f"(out[46]),
          "=f"(out[47]),
          "=f"(out[48]),
          "=f"(out[49]),
          "=f"(out[50]),
          "=f"(out[51]),
          "=f"(out[52]),
          "=f"(out[53]),
          "=f"(out[54]),
          "=f"(out[55]),
          "=f"(out[56]),
          "=f"(out[57]),
          "=f"(out[58]),
          "=f"(out[59]),
          "=f"(out[60]),
          "=f"(out[61]),
          "=f"(out[62]),
          "=f"(out[63]),
          "=f"(out[64]),
          "=f"(out[65]),
          "=f"(out[66]),
          "=f"(out[67]),
          "=f"(out[68]),
          "=f"(out[69]),
          "=f"(out[70]),
          "=f"(out[71]),
          "=f"(out[72]),
          "=f"(out[73]),
          "=f"(out[74]),
          "=f"(out[75]),
          "=f"(out[76]),
          "=f"(out[77]),
          "=f"(out[78]),
          "=f"(out[79]),
          "=f"(out[80]),
          "=f"(out[81]),
          "=f"(out[82]),
          "=f"(out[83]),
          "=f"(out[84]),
          "=f"(out[85]),
          "=f"(out[86]),
          "=f"(out[87]),
          "=f"(out[88]),
          "=f"(out[89]),
          "=f"(out[90]),
          "=f"(out[91]),
          "=f"(out[92]),
          "=f"(out[93]),
          "=f"(out[94]),
          "=f"(out[95]),
          "=f"(out[96]),
          "=f"(out[97]),
          "=f"(out[98]),
          "=f"(out[99]),
          "=f"(out[100]),
          "=f"(out[101]),
          "=f"(out[102]),
          "=f"(out[103]),
          "=f"(out[104]),
          "=f"(out[105]),
          "=f"(out[106]),
          "=f"(out[107]),
          "=f"(out[108]),
          "=f"(out[109]),
          "=f"(out[110]),
          "=f"(out[111]),
          "=f"(out[112]),
          "=f"(out[113]),
          "=f"(out[114]),
          "=f"(out[115]),
          "=f"(out[116]),
          "=f"(out[117]),
          "=f"(out[118]),
          "=f"(out[119]),
          "=f"(out[120]),
          "=f"(out[121]),
          "=f"(out[122]),
          "=f"(out[123]),
          "=f"(out[124]),
          "=f"(out[125]),
          "=f"(out[126]),
          "=f"(out[127]),
          "=f"(redval)
        : "r"(taddr)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_32x32b_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x2.u32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[2],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[2],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::uint32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x2.u32.min {%0, %1}, %2, [%3], %4;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x2.u32.max {%0, %1}, %2, [%3], %4;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x2.s32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[2],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[2],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::int32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x2.s32.min {%0, %1}, %2, [%3], %4;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x2.s32.max {%0, %1}, %2, [%3], %4;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x2.f32.op.abs out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[2],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[2],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x2.f32.min.abs {%0, %1}, %2, [%3], %4;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x2.f32.max.abs {%0, %1}, %2, [%3], %4;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x2.f32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  float (&out)[2],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  float (&out)[2],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x2.f32.min {%0, %1}, %2, [%3], %4;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x2.f32.max {%0, %1}, %2, [%3], %4;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x4.u32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[4],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[4],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::uint32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x4.u32.min {%0, %1, %2, %3}, %4, [%5], %6;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x4.u32.max {%0, %1, %2, %3}, %4, [%5], %6;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x4.s32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[4],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[4],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::int32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x4.s32.min {%0, %1, %2, %3}, %4, [%5], %6;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x4.s32.max {%0, %1, %2, %3}, %4, [%5], %6;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x4.f32.op.abs out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[4],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[4],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x4.f32.min.abs {%0, %1, %2, %3}, %4, [%5], %6;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x4.f32.max.abs {%0, %1, %2, %3}, %4, [%5], %6;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x4.f32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  float (&out)[4],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  float (&out)[4],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x4.f32.min {%0, %1, %2, %3}, %4, [%5], %6;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x4.f32.max {%0, %1, %2, %3}, %4, [%5], %6;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x8.u32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[8],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[8],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::uint32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x8.u32.min {%0, %1, %2, %3, %4, %5, %6, %7}, %8, [%9], %10;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x8.u32.max {%0, %1, %2, %3, %4, %5, %6, %7}, %8, [%9], %10;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x8.s32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[8],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[8],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::int32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x8.s32.min {%0, %1, %2, %3, %4, %5, %6, %7}, %8, [%9], %10;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x8.s32.max {%0, %1, %2, %3, %4, %5, %6, %7}, %8, [%9], %10;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x8.f32.op.abs out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[8],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[8],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x8.f32.min.abs {%0, %1, %2, %3, %4, %5, %6, %7}, %8, [%9], %10;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x8.f32.max.abs {%0, %1, %2, %3, %4, %5, %6, %7}, %8, [%9], %10;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x8.f32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  float (&out)[8],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  float (&out)[8],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x8.f32.min {%0, %1, %2, %3, %4, %5, %6, %7}, %8, [%9], %10;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x8.f32.max {%0, %1, %2, %3, %4, %5, %6, %7}, %8, [%9], %10;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x16.u32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[16],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[16],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::uint32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x16.u32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, %16, [%17], %18;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x16.u32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, %16, [%17], %18;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x16.s32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[16],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[16],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::int32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x16.s32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, %16, [%17], %18;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x16.s32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, %16, [%17], %18;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x16.f32.op.abs out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[16],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[16],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x16.f32.min.abs {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, %16, [%17], %18;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x16.f32.max.abs {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, %16, [%17], %18;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x16.f32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  float (&out)[16],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  float (&out)[16],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x16.f32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, %16, [%17], %18;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x16.f32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15}, %16, [%17], %18;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x32.u32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[32],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[32],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::uint32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x32.u32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, %32, [%33], %34;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x32.u32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, %32, [%33], %34;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x32.s32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[32],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[32],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::int32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x32.s32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, %32, [%33], %34;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x32.s32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, %32, [%33], %34;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x32.f32.op.abs out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[32],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[32],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x32.f32.min.abs {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, %32, [%33], %34;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x32.f32.max.abs {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, %32, [%33], %34;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x32.f32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  float (&out)[32],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  float (&out)[32],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x32.f32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, %32, [%33], %34;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x32.f32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31}, %32, [%33], %34;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x64.u32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[64],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[64],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::uint32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x64.u32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, %64, [%65], %66;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(out[32]),
          "=r"(out[33]),
          "=r"(out[34]),
          "=r"(out[35]),
          "=r"(out[36]),
          "=r"(out[37]),
          "=r"(out[38]),
          "=r"(out[39]),
          "=r"(out[40]),
          "=r"(out[41]),
          "=r"(out[42]),
          "=r"(out[43]),
          "=r"(out[44]),
          "=r"(out[45]),
          "=r"(out[46]),
          "=r"(out[47]),
          "=r"(out[48]),
          "=r"(out[49]),
          "=r"(out[50]),
          "=r"(out[51]),
          "=r"(out[52]),
          "=r"(out[53]),
          "=r"(out[54]),
          "=r"(out[55]),
          "=r"(out[56]),
          "=r"(out[57]),
          "=r"(out[58]),
          "=r"(out[59]),
          "=r"(out[60]),
          "=r"(out[61]),
          "=r"(out[62]),
          "=r"(out[63]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x64.u32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, %64, [%65], %66;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(out[32]),
          "=r"(out[33]),
          "=r"(out[34]),
          "=r"(out[35]),
          "=r"(out[36]),
          "=r"(out[37]),
          "=r"(out[38]),
          "=r"(out[39]),
          "=r"(out[40]),
          "=r"(out[41]),
          "=r"(out[42]),
          "=r"(out[43]),
          "=r"(out[44]),
          "=r"(out[45]),
          "=r"(out[46]),
          "=r"(out[47]),
          "=r"(out[48]),
          "=r"(out[49]),
          "=r"(out[50]),
          "=r"(out[51]),
          "=r"(out[52]),
          "=r"(out[53]),
          "=r"(out[54]),
          "=r"(out[55]),
          "=r"(out[56]),
          "=r"(out[57]),
          "=r"(out[58]),
          "=r"(out[59]),
          "=r"(out[60]),
          "=r"(out[61]),
          "=r"(out[62]),
          "=r"(out[63]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x64.s32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[64],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[64],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::int32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x64.s32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, %64, [%65], %66;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(out[32]),
          "=r"(out[33]),
          "=r"(out[34]),
          "=r"(out[35]),
          "=r"(out[36]),
          "=r"(out[37]),
          "=r"(out[38]),
          "=r"(out[39]),
          "=r"(out[40]),
          "=r"(out[41]),
          "=r"(out[42]),
          "=r"(out[43]),
          "=r"(out[44]),
          "=r"(out[45]),
          "=r"(out[46]),
          "=r"(out[47]),
          "=r"(out[48]),
          "=r"(out[49]),
          "=r"(out[50]),
          "=r"(out[51]),
          "=r"(out[52]),
          "=r"(out[53]),
          "=r"(out[54]),
          "=r"(out[55]),
          "=r"(out[56]),
          "=r"(out[57]),
          "=r"(out[58]),
          "=r"(out[59]),
          "=r"(out[60]),
          "=r"(out[61]),
          "=r"(out[62]),
          "=r"(out[63]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x64.s32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, %64, [%65], %66;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(out[32]),
          "=r"(out[33]),
          "=r"(out[34]),
          "=r"(out[35]),
          "=r"(out[36]),
          "=r"(out[37]),
          "=r"(out[38]),
          "=r"(out[39]),
          "=r"(out[40]),
          "=r"(out[41]),
          "=r"(out[42]),
          "=r"(out[43]),
          "=r"(out[44]),
          "=r"(out[45]),
          "=r"(out[46]),
          "=r"(out[47]),
          "=r"(out[48]),
          "=r"(out[49]),
          "=r"(out[50]),
          "=r"(out[51]),
          "=r"(out[52]),
          "=r"(out[53]),
          "=r"(out[54]),
          "=r"(out[55]),
          "=r"(out[56]),
          "=r"(out[57]),
          "=r"(out[58]),
          "=r"(out[59]),
          "=r"(out[60]),
          "=r"(out[61]),
          "=r"(out[62]),
          "=r"(out[63]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x64.f32.op.abs out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[64],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[64],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x64.f32.min.abs {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, %64, [%65], %66;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(out[32]),
          "=f"(out[33]),
          "=f"(out[34]),
          "=f"(out[35]),
          "=f"(out[36]),
          "=f"(out[37]),
          "=f"(out[38]),
          "=f"(out[39]),
          "=f"(out[40]),
          "=f"(out[41]),
          "=f"(out[42]),
          "=f"(out[43]),
          "=f"(out[44]),
          "=f"(out[45]),
          "=f"(out[46]),
          "=f"(out[47]),
          "=f"(out[48]),
          "=f"(out[49]),
          "=f"(out[50]),
          "=f"(out[51]),
          "=f"(out[52]),
          "=f"(out[53]),
          "=f"(out[54]),
          "=f"(out[55]),
          "=f"(out[56]),
          "=f"(out[57]),
          "=f"(out[58]),
          "=f"(out[59]),
          "=f"(out[60]),
          "=f"(out[61]),
          "=f"(out[62]),
          "=f"(out[63]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x64.f32.max.abs {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, %64, [%65], %66;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(out[32]),
          "=f"(out[33]),
          "=f"(out[34]),
          "=f"(out[35]),
          "=f"(out[36]),
          "=f"(out[37]),
          "=f"(out[38]),
          "=f"(out[39]),
          "=f"(out[40]),
          "=f"(out[41]),
          "=f"(out[42]),
          "=f"(out[43]),
          "=f"(out[44]),
          "=f"(out[45]),
          "=f"(out[46]),
          "=f"(out[47]),
          "=f"(out[48]),
          "=f"(out[49]),
          "=f"(out[50]),
          "=f"(out[51]),
          "=f"(out[52]),
          "=f"(out[53]),
          "=f"(out[54]),
          "=f"(out[55]),
          "=f"(out[56]),
          "=f"(out[57]),
          "=f"(out[58]),
          "=f"(out[59]),
          "=f"(out[60]),
          "=f"(out[61]),
          "=f"(out[62]),
          "=f"(out[63]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x64.f32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  float (&out)[64],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  float (&out)[64],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x64.f32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, %64, [%65], %66;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(out[32]),
          "=f"(out[33]),
          "=f"(out[34]),
          "=f"(out[35]),
          "=f"(out[36]),
          "=f"(out[37]),
          "=f"(out[38]),
          "=f"(out[39]),
          "=f"(out[40]),
          "=f"(out[41]),
          "=f"(out[42]),
          "=f"(out[43]),
          "=f"(out[44]),
          "=f"(out[45]),
          "=f"(out[46]),
          "=f"(out[47]),
          "=f"(out[48]),
          "=f"(out[49]),
          "=f"(out[50]),
          "=f"(out[51]),
          "=f"(out[52]),
          "=f"(out[53]),
          "=f"(out[54]),
          "=f"(out[55]),
          "=f"(out[56]),
          "=f"(out[57]),
          "=f"(out[58]),
          "=f"(out[59]),
          "=f"(out[60]),
          "=f"(out[61]),
          "=f"(out[62]),
          "=f"(out[63]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x64.f32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63}, %64, [%65], %66;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(out[32]),
          "=f"(out[33]),
          "=f"(out[34]),
          "=f"(out[35]),
          "=f"(out[36]),
          "=f"(out[37]),
          "=f"(out[38]),
          "=f"(out[39]),
          "=f"(out[40]),
          "=f"(out[41]),
          "=f"(out[42]),
          "=f"(out[43]),
          "=f"(out[44]),
          "=f"(out[45]),
          "=f"(out[46]),
          "=f"(out[47]),
          "=f"(out[48]),
          "=f"(out[49]),
          "=f"(out[50]),
          "=f"(out[51]),
          "=f"(out[52]),
          "=f"(out[53]),
          "=f"(out[54]),
          "=f"(out[55]),
          "=f"(out[56]),
          "=f"(out[57]),
          "=f"(out[58]),
          "=f"(out[59]),
          "=f"(out[60]),
          "=f"(out[61]),
          "=f"(out[62]),
          "=f"(out[63]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x128.u32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[128],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::uint32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::uint32_t (&out)[128],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::uint32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x128.u32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, %128, [%129], %130;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(out[32]),
          "=r"(out[33]),
          "=r"(out[34]),
          "=r"(out[35]),
          "=r"(out[36]),
          "=r"(out[37]),
          "=r"(out[38]),
          "=r"(out[39]),
          "=r"(out[40]),
          "=r"(out[41]),
          "=r"(out[42]),
          "=r"(out[43]),
          "=r"(out[44]),
          "=r"(out[45]),
          "=r"(out[46]),
          "=r"(out[47]),
          "=r"(out[48]),
          "=r"(out[49]),
          "=r"(out[50]),
          "=r"(out[51]),
          "=r"(out[52]),
          "=r"(out[53]),
          "=r"(out[54]),
          "=r"(out[55]),
          "=r"(out[56]),
          "=r"(out[57]),
          "=r"(out[58]),
          "=r"(out[59]),
          "=r"(out[60]),
          "=r"(out[61]),
          "=r"(out[62]),
          "=r"(out[63]),
          "=r"(out[64]),
          "=r"(out[65]),
          "=r"(out[66]),
          "=r"(out[67]),
          "=r"(out[68]),
          "=r"(out[69]),
          "=r"(out[70]),
          "=r"(out[71]),
          "=r"(out[72]),
          "=r"(out[73]),
          "=r"(out[74]),
          "=r"(out[75]),
          "=r"(out[76]),
          "=r"(out[77]),
          "=r"(out[78]),
          "=r"(out[79]),
          "=r"(out[80]),
          "=r"(out[81]),
          "=r"(out[82]),
          "=r"(out[83]),
          "=r"(out[84]),
          "=r"(out[85]),
          "=r"(out[86]),
          "=r"(out[87]),
          "=r"(out[88]),
          "=r"(out[89]),
          "=r"(out[90]),
          "=r"(out[91]),
          "=r"(out[92]),
          "=r"(out[93]),
          "=r"(out[94]),
          "=r"(out[95]),
          "=r"(out[96]),
          "=r"(out[97]),
          "=r"(out[98]),
          "=r"(out[99]),
          "=r"(out[100]),
          "=r"(out[101]),
          "=r"(out[102]),
          "=r"(out[103]),
          "=r"(out[104]),
          "=r"(out[105]),
          "=r"(out[106]),
          "=r"(out[107]),
          "=r"(out[108]),
          "=r"(out[109]),
          "=r"(out[110]),
          "=r"(out[111]),
          "=r"(out[112]),
          "=r"(out[113]),
          "=r"(out[114]),
          "=r"(out[115]),
          "=r"(out[116]),
          "=r"(out[117]),
          "=r"(out[118]),
          "=r"(out[119]),
          "=r"(out[120]),
          "=r"(out[121]),
          "=r"(out[122]),
          "=r"(out[123]),
          "=r"(out[124]),
          "=r"(out[125]),
          "=r"(out[126]),
          "=r"(out[127]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x128.u32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, %128, [%129], %130;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(out[32]),
          "=r"(out[33]),
          "=r"(out[34]),
          "=r"(out[35]),
          "=r"(out[36]),
          "=r"(out[37]),
          "=r"(out[38]),
          "=r"(out[39]),
          "=r"(out[40]),
          "=r"(out[41]),
          "=r"(out[42]),
          "=r"(out[43]),
          "=r"(out[44]),
          "=r"(out[45]),
          "=r"(out[46]),
          "=r"(out[47]),
          "=r"(out[48]),
          "=r"(out[49]),
          "=r"(out[50]),
          "=r"(out[51]),
          "=r"(out[52]),
          "=r"(out[53]),
          "=r"(out[54]),
          "=r"(out[55]),
          "=r"(out[56]),
          "=r"(out[57]),
          "=r"(out[58]),
          "=r"(out[59]),
          "=r"(out[60]),
          "=r"(out[61]),
          "=r"(out[62]),
          "=r"(out[63]),
          "=r"(out[64]),
          "=r"(out[65]),
          "=r"(out[66]),
          "=r"(out[67]),
          "=r"(out[68]),
          "=r"(out[69]),
          "=r"(out[70]),
          "=r"(out[71]),
          "=r"(out[72]),
          "=r"(out[73]),
          "=r"(out[74]),
          "=r"(out[75]),
          "=r"(out[76]),
          "=r"(out[77]),
          "=r"(out[78]),
          "=r"(out[79]),
          "=r"(out[80]),
          "=r"(out[81]),
          "=r"(out[82]),
          "=r"(out[83]),
          "=r"(out[84]),
          "=r"(out[85]),
          "=r"(out[86]),
          "=r"(out[87]),
          "=r"(out[88]),
          "=r"(out[89]),
          "=r"(out[90]),
          "=r"(out[91]),
          "=r"(out[92]),
          "=r"(out[93]),
          "=r"(out[94]),
          "=r"(out[95]),
          "=r"(out[96]),
          "=r"(out[97]),
          "=r"(out[98]),
          "=r"(out[99]),
          "=r"(out[100]),
          "=r"(out[101]),
          "=r"(out[102]),
          "=r"(out[103]),
          "=r"(out[104]),
          "=r"(out[105]),
          "=r"(out[106]),
          "=r"(out[107]),
          "=r"(out[108]),
          "=r"(out[109]),
          "=r"(out[110]),
          "=r"(out[111]),
          "=r"(out[112]),
          "=r"(out[113]),
          "=r"(out[114]),
          "=r"(out[115]),
          "=r"(out[116]),
          "=r"(out[117]),
          "=r"(out[118]),
          "=r"(out[119]),
          "=r"(out[120]),
          "=r"(out[121]),
          "=r"(out[122]),
          "=r"(out[123]),
          "=r"(out[124]),
          "=r"(out[125]),
          "=r"(out[126]),
          "=r"(out[127]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x128.s32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[128],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline std::int32_t tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  std::int32_t (&out)[128],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    std::int32_t redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x128.s32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, %128, [%129], %130;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(out[32]),
          "=r"(out[33]),
          "=r"(out[34]),
          "=r"(out[35]),
          "=r"(out[36]),
          "=r"(out[37]),
          "=r"(out[38]),
          "=r"(out[39]),
          "=r"(out[40]),
          "=r"(out[41]),
          "=r"(out[42]),
          "=r"(out[43]),
          "=r"(out[44]),
          "=r"(out[45]),
          "=r"(out[46]),
          "=r"(out[47]),
          "=r"(out[48]),
          "=r"(out[49]),
          "=r"(out[50]),
          "=r"(out[51]),
          "=r"(out[52]),
          "=r"(out[53]),
          "=r"(out[54]),
          "=r"(out[55]),
          "=r"(out[56]),
          "=r"(out[57]),
          "=r"(out[58]),
          "=r"(out[59]),
          "=r"(out[60]),
          "=r"(out[61]),
          "=r"(out[62]),
          "=r"(out[63]),
          "=r"(out[64]),
          "=r"(out[65]),
          "=r"(out[66]),
          "=r"(out[67]),
          "=r"(out[68]),
          "=r"(out[69]),
          "=r"(out[70]),
          "=r"(out[71]),
          "=r"(out[72]),
          "=r"(out[73]),
          "=r"(out[74]),
          "=r"(out[75]),
          "=r"(out[76]),
          "=r"(out[77]),
          "=r"(out[78]),
          "=r"(out[79]),
          "=r"(out[80]),
          "=r"(out[81]),
          "=r"(out[82]),
          "=r"(out[83]),
          "=r"(out[84]),
          "=r"(out[85]),
          "=r"(out[86]),
          "=r"(out[87]),
          "=r"(out[88]),
          "=r"(out[89]),
          "=r"(out[90]),
          "=r"(out[91]),
          "=r"(out[92]),
          "=r"(out[93]),
          "=r"(out[94]),
          "=r"(out[95]),
          "=r"(out[96]),
          "=r"(out[97]),
          "=r"(out[98]),
          "=r"(out[99]),
          "=r"(out[100]),
          "=r"(out[101]),
          "=r"(out[102]),
          "=r"(out[103]),
          "=r"(out[104]),
          "=r"(out[105]),
          "=r"(out[106]),
          "=r"(out[107]),
          "=r"(out[108]),
          "=r"(out[109]),
          "=r"(out[110]),
          "=r"(out[111]),
          "=r"(out[112]),
          "=r"(out[113]),
          "=r"(out[114]),
          "=r"(out[115]),
          "=r"(out[116]),
          "=r"(out[117]),
          "=r"(out[118]),
          "=r"(out[119]),
          "=r"(out[120]),
          "=r"(out[121]),
          "=r"(out[122]),
          "=r"(out[123]),
          "=r"(out[124]),
          "=r"(out[125]),
          "=r"(out[126]),
          "=r"(out[127]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x128.s32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, %128, [%129], %130;"
        : "=r"(out[0]),
          "=r"(out[1]),
          "=r"(out[2]),
          "=r"(out[3]),
          "=r"(out[4]),
          "=r"(out[5]),
          "=r"(out[6]),
          "=r"(out[7]),
          "=r"(out[8]),
          "=r"(out[9]),
          "=r"(out[10]),
          "=r"(out[11]),
          "=r"(out[12]),
          "=r"(out[13]),
          "=r"(out[14]),
          "=r"(out[15]),
          "=r"(out[16]),
          "=r"(out[17]),
          "=r"(out[18]),
          "=r"(out[19]),
          "=r"(out[20]),
          "=r"(out[21]),
          "=r"(out[22]),
          "=r"(out[23]),
          "=r"(out[24]),
          "=r"(out[25]),
          "=r"(out[26]),
          "=r"(out[27]),
          "=r"(out[28]),
          "=r"(out[29]),
          "=r"(out[30]),
          "=r"(out[31]),
          "=r"(out[32]),
          "=r"(out[33]),
          "=r"(out[34]),
          "=r"(out[35]),
          "=r"(out[36]),
          "=r"(out[37]),
          "=r"(out[38]),
          "=r"(out[39]),
          "=r"(out[40]),
          "=r"(out[41]),
          "=r"(out[42]),
          "=r"(out[43]),
          "=r"(out[44]),
          "=r"(out[45]),
          "=r"(out[46]),
          "=r"(out[47]),
          "=r"(out[48]),
          "=r"(out[49]),
          "=r"(out[50]),
          "=r"(out[51]),
          "=r"(out[52]),
          "=r"(out[53]),
          "=r"(out[54]),
          "=r"(out[55]),
          "=r"(out[56]),
          "=r"(out[57]),
          "=r"(out[58]),
          "=r"(out[59]),
          "=r"(out[60]),
          "=r"(out[61]),
          "=r"(out[62]),
          "=r"(out[63]),
          "=r"(out[64]),
          "=r"(out[65]),
          "=r"(out[66]),
          "=r"(out[67]),
          "=r"(out[68]),
          "=r"(out[69]),
          "=r"(out[70]),
          "=r"(out[71]),
          "=r"(out[72]),
          "=r"(out[73]),
          "=r"(out[74]),
          "=r"(out[75]),
          "=r"(out[76]),
          "=r"(out[77]),
          "=r"(out[78]),
          "=r"(out[79]),
          "=r"(out[80]),
          "=r"(out[81]),
          "=r"(out[82]),
          "=r"(out[83]),
          "=r"(out[84]),
          "=r"(out[85]),
          "=r"(out[86]),
          "=r"(out[87]),
          "=r"(out[88]),
          "=r"(out[89]),
          "=r"(out[90]),
          "=r"(out[91]),
          "=r"(out[92]),
          "=r"(out[93]),
          "=r"(out[94]),
          "=r"(out[95]),
          "=r"(out[96]),
          "=r"(out[97]),
          "=r"(out[98]),
          "=r"(out[99]),
          "=r"(out[100]),
          "=r"(out[101]),
          "=r"(out[102]),
          "=r"(out[103]),
          "=r"(out[104]),
          "=r"(out[105]),
          "=r"(out[106]),
          "=r"(out[107]),
          "=r"(out[108]),
          "=r"(out[109]),
          "=r"(out[110]),
          "=r"(out[111]),
          "=r"(out[112]),
          "=r"(out[113]),
          "=r"(out[114]),
          "=r"(out[115]),
          "=r"(out[116]),
          "=r"(out[117]),
          "=r"(out[118]),
          "=r"(out[119]),
          "=r"(out[120]),
          "=r"(out[121]),
          "=r"(out[122]),
          "=r"(out[123]),
          "=r"(out[124]),
          "=r"(out[125]),
          "=r"(out[126]),
          "=r"(out[127]),
          "=r"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x128.f32.op.abs out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[128],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2_abs(
  cuda_ptx::op_t<Op> op,
  float (&out)[128],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x128.f32.min.abs {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, %128, [%129], %130;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(out[32]),
          "=f"(out[33]),
          "=f"(out[34]),
          "=f"(out[35]),
          "=f"(out[36]),
          "=f"(out[37]),
          "=f"(out[38]),
          "=f"(out[39]),
          "=f"(out[40]),
          "=f"(out[41]),
          "=f"(out[42]),
          "=f"(out[43]),
          "=f"(out[44]),
          "=f"(out[45]),
          "=f"(out[46]),
          "=f"(out[47]),
          "=f"(out[48]),
          "=f"(out[49]),
          "=f"(out[50]),
          "=f"(out[51]),
          "=f"(out[52]),
          "=f"(out[53]),
          "=f"(out[54]),
          "=f"(out[55]),
          "=f"(out[56]),
          "=f"(out[57]),
          "=f"(out[58]),
          "=f"(out[59]),
          "=f"(out[60]),
          "=f"(out[61]),
          "=f"(out[62]),
          "=f"(out[63]),
          "=f"(out[64]),
          "=f"(out[65]),
          "=f"(out[66]),
          "=f"(out[67]),
          "=f"(out[68]),
          "=f"(out[69]),
          "=f"(out[70]),
          "=f"(out[71]),
          "=f"(out[72]),
          "=f"(out[73]),
          "=f"(out[74]),
          "=f"(out[75]),
          "=f"(out[76]),
          "=f"(out[77]),
          "=f"(out[78]),
          "=f"(out[79]),
          "=f"(out[80]),
          "=f"(out[81]),
          "=f"(out[82]),
          "=f"(out[83]),
          "=f"(out[84]),
          "=f"(out[85]),
          "=f"(out[86]),
          "=f"(out[87]),
          "=f"(out[88]),
          "=f"(out[89]),
          "=f"(out[90]),
          "=f"(out[91]),
          "=f"(out[92]),
          "=f"(out[93]),
          "=f"(out[94]),
          "=f"(out[95]),
          "=f"(out[96]),
          "=f"(out[97]),
          "=f"(out[98]),
          "=f"(out[99]),
          "=f"(out[100]),
          "=f"(out[101]),
          "=f"(out[102]),
          "=f"(out[103]),
          "=f"(out[104]),
          "=f"(out[105]),
          "=f"(out[106]),
          "=f"(out[107]),
          "=f"(out[108]),
          "=f"(out[109]),
          "=f"(out[110]),
          "=f"(out[111]),
          "=f"(out[112]),
          "=f"(out[113]),
          "=f"(out[114]),
          "=f"(out[115]),
          "=f"(out[116]),
          "=f"(out[117]),
          "=f"(out[118]),
          "=f"(out[119]),
          "=f"(out[120]),
          "=f"(out[121]),
          "=f"(out[122]),
          "=f"(out[123]),
          "=f"(out[124]),
          "=f"(out[125]),
          "=f"(out[126]),
          "=f"(out[127]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x128.f32.max.abs {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, %128, [%129], %130;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(out[32]),
          "=f"(out[33]),
          "=f"(out[34]),
          "=f"(out[35]),
          "=f"(out[36]),
          "=f"(out[37]),
          "=f"(out[38]),
          "=f"(out[39]),
          "=f"(out[40]),
          "=f"(out[41]),
          "=f"(out[42]),
          "=f"(out[43]),
          "=f"(out[44]),
          "=f"(out[45]),
          "=f"(out[46]),
          "=f"(out[47]),
          "=f"(out[48]),
          "=f"(out[49]),
          "=f"(out[50]),
          "=f"(out[51]),
          "=f"(out[52]),
          "=f"(out[53]),
          "=f"(out[54]),
          "=f"(out[55]),
          "=f"(out[56]),
          "=f"(out[57]),
          "=f"(out[58]),
          "=f"(out[59]),
          "=f"(out[60]),
          "=f"(out[61]),
          "=f"(out[62]),
          "=f"(out[63]),
          "=f"(out[64]),
          "=f"(out[65]),
          "=f"(out[66]),
          "=f"(out[67]),
          "=f"(out[68]),
          "=f"(out[69]),
          "=f"(out[70]),
          "=f"(out[71]),
          "=f"(out[72]),
          "=f"(out[73]),
          "=f"(out[74]),
          "=f"(out[75]),
          "=f"(out[76]),
          "=f"(out[77]),
          "=f"(out[78]),
          "=f"(out[79]),
          "=f"(out[80]),
          "=f"(out[81]),
          "=f"(out[82]),
          "=f"(out[83]),
          "=f"(out[84]),
          "=f"(out[85]),
          "=f"(out[86]),
          "=f"(out[87]),
          "=f"(out[88]),
          "=f"(out[89]),
          "=f"(out[90]),
          "=f"(out[91]),
          "=f"(out[92]),
          "=f"(out[93]),
          "=f"(out[94]),
          "=f"(out[95]),
          "=f"(out[96]),
          "=f"(out[97]),
          "=f"(out[98]),
          "=f"(out[99]),
          "=f"(out[100]),
          "=f"(out[101]),
          "=f"(out[102]),
          "=f"(out[103]),
          "=f"(out[104]),
          "=f"(out[105]),
          "=f"(out[106]),
          "=f"(out[107]),
          "=f"(out[108]),
          "=f"(out[109]),
          "=f"(out[110]),
          "=f"(out[111]),
          "=f"(out[112]),
          "=f"(out[113]),
          "=f"(out[114]),
          "=f"(out[115]),
          "=f"(out[116]),
          "=f"(out[117]),
          "=f"(out[118]),
          "=f"(out[119]),
          "=f"(out[120]),
          "=f"(out[121]),
          "=f"(out[122]),
          "=f"(out[123]),
          "=f"(out[124]),
          "=f"(out[125]),
          "=f"(out[126]),
          "=f"(out[127]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_abs_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.ld.red.sync.aligned.16x32bx2.x128.f32.op out, redval, [taddr], immHalfSplitoff; // PTX ISA 88, SM_103a, SM_103f, SM_110a, SM_110f
// .op        = { .min, .max }
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  float (&out)[128],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_op Op>
__device__ static inline float tcgen05_ld_red_16x32bx2(
  cuda_ptx::op_t<Op> op,
  float (&out)[128],
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff)
{
  static_assert(op == op_min || op == op_max, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    float redval;
    if constexpr (op == op_min) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x128.f32.min {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, %128, [%129], %130;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(out[32]),
          "=f"(out[33]),
          "=f"(out[34]),
          "=f"(out[35]),
          "=f"(out[36]),
          "=f"(out[37]),
          "=f"(out[38]),
          "=f"(out[39]),
          "=f"(out[40]),
          "=f"(out[41]),
          "=f"(out[42]),
          "=f"(out[43]),
          "=f"(out[44]),
          "=f"(out[45]),
          "=f"(out[46]),
          "=f"(out[47]),
          "=f"(out[48]),
          "=f"(out[49]),
          "=f"(out[50]),
          "=f"(out[51]),
          "=f"(out[52]),
          "=f"(out[53]),
          "=f"(out[54]),
          "=f"(out[55]),
          "=f"(out[56]),
          "=f"(out[57]),
          "=f"(out[58]),
          "=f"(out[59]),
          "=f"(out[60]),
          "=f"(out[61]),
          "=f"(out[62]),
          "=f"(out[63]),
          "=f"(out[64]),
          "=f"(out[65]),
          "=f"(out[66]),
          "=f"(out[67]),
          "=f"(out[68]),
          "=f"(out[69]),
          "=f"(out[70]),
          "=f"(out[71]),
          "=f"(out[72]),
          "=f"(out[73]),
          "=f"(out[74]),
          "=f"(out[75]),
          "=f"(out[76]),
          "=f"(out[77]),
          "=f"(out[78]),
          "=f"(out[79]),
          "=f"(out[80]),
          "=f"(out[81]),
          "=f"(out[82]),
          "=f"(out[83]),
          "=f"(out[84]),
          "=f"(out[85]),
          "=f"(out[86]),
          "=f"(out[87]),
          "=f"(out[88]),
          "=f"(out[89]),
          "=f"(out[90]),
          "=f"(out[91]),
          "=f"(out[92]),
          "=f"(out[93]),
          "=f"(out[94]),
          "=f"(out[95]),
          "=f"(out[96]),
          "=f"(out[97]),
          "=f"(out[98]),
          "=f"(out[99]),
          "=f"(out[100]),
          "=f"(out[101]),
          "=f"(out[102]),
          "=f"(out[103]),
          "=f"(out[104]),
          "=f"(out[105]),
          "=f"(out[106]),
          "=f"(out[107]),
          "=f"(out[108]),
          "=f"(out[109]),
          "=f"(out[110]),
          "=f"(out[111]),
          "=f"(out[112]),
          "=f"(out[113]),
          "=f"(out[114]),
          "=f"(out[115]),
          "=f"(out[116]),
          "=f"(out[117]),
          "=f"(out[118]),
          "=f"(out[119]),
          "=f"(out[120]),
          "=f"(out[121]),
          "=f"(out[122]),
          "=f"(out[123]),
          "=f"(out[124]),
          "=f"(out[125]),
          "=f"(out[126]),
          "=f"(out[127]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    } else if constexpr (op == op_max) {
      asm (
        "tcgen05.ld.red.sync.aligned.16x32bx2.x128.f32.max {%0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127}, %128, [%129], %130;"
        : "=f"(out[0]),
          "=f"(out[1]),
          "=f"(out[2]),
          "=f"(out[3]),
          "=f"(out[4]),
          "=f"(out[5]),
          "=f"(out[6]),
          "=f"(out[7]),
          "=f"(out[8]),
          "=f"(out[9]),
          "=f"(out[10]),
          "=f"(out[11]),
          "=f"(out[12]),
          "=f"(out[13]),
          "=f"(out[14]),
          "=f"(out[15]),
          "=f"(out[16]),
          "=f"(out[17]),
          "=f"(out[18]),
          "=f"(out[19]),
          "=f"(out[20]),
          "=f"(out[21]),
          "=f"(out[22]),
          "=f"(out[23]),
          "=f"(out[24]),
          "=f"(out[25]),
          "=f"(out[26]),
          "=f"(out[27]),
          "=f"(out[28]),
          "=f"(out[29]),
          "=f"(out[30]),
          "=f"(out[31]),
          "=f"(out[32]),
          "=f"(out[33]),
          "=f"(out[34]),
          "=f"(out[35]),
          "=f"(out[36]),
          "=f"(out[37]),
          "=f"(out[38]),
          "=f"(out[39]),
          "=f"(out[40]),
          "=f"(out[41]),
          "=f"(out[42]),
          "=f"(out[43]),
          "=f"(out[44]),
          "=f"(out[45]),
          "=f"(out[46]),
          "=f"(out[47]),
          "=f"(out[48]),
          "=f"(out[49]),
          "=f"(out[50]),
          "=f"(out[51]),
          "=f"(out[52]),
          "=f"(out[53]),
          "=f"(out[54]),
          "=f"(out[55]),
          "=f"(out[56]),
          "=f"(out[57]),
          "=f"(out[58]),
          "=f"(out[59]),
          "=f"(out[60]),
          "=f"(out[61]),
          "=f"(out[62]),
          "=f"(out[63]),
          "=f"(out[64]),
          "=f"(out[65]),
          "=f"(out[66]),
          "=f"(out[67]),
          "=f"(out[68]),
          "=f"(out[69]),
          "=f"(out[70]),
          "=f"(out[71]),
          "=f"(out[72]),
          "=f"(out[73]),
          "=f"(out[74]),
          "=f"(out[75]),
          "=f"(out[76]),
          "=f"(out[77]),
          "=f"(out[78]),
          "=f"(out[79]),
          "=f"(out[80]),
          "=f"(out[81]),
          "=f"(out[82]),
          "=f"(out[83]),
          "=f"(out[84]),
          "=f"(out[85]),
          "=f"(out[86]),
          "=f"(out[87]),
          "=f"(out[88]),
          "=f"(out[89]),
          "=f"(out[90]),
          "=f"(out[91]),
          "=f"(out[92]),
          "=f"(out[93]),
          "=f"(out[94]),
          "=f"(out[95]),
          "=f"(out[96]),
          "=f"(out[97]),
          "=f"(out[98]),
          "=f"(out[99]),
          "=f"(out[100]),
          "=f"(out[101]),
          "=f"(out[102]),
          "=f"(out[103]),
          "=f"(out[104]),
          "=f"(out[105]),
          "=f"(out[106]),
          "=f"(out[107]),
          "=f"(out[108]),
          "=f"(out[109]),
          "=f"(out[110]),
          "=f"(out[111]),
          "=f"(out[112]),
          "=f"(out[113]),
          "=f"(out[114]),
          "=f"(out[115]),
          "=f"(out[116]),
          "=f"(out[117]),
          "=f"(out[118]),
          "=f"(out[119]),
          "=f"(out[120]),
          "=f"(out[121]),
          "=f"(out[122]),
          "=f"(out[123]),
          "=f"(out[124]),
          "=f"(out[125]),
          "=f"(out[126]),
          "=f"(out[127]),
          "=f"(redval)
        : "r"(taddr),
          "n"(immHalfSplitoff.value)
        : "memory"
      );
    }
    return redval;

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_ld_red_16x32bx2_is_only_supported_on_SM_103a_103f_110a_110f_depending_on_the_variant__();
    return 0;
  #endif
}
#endif // __libcuda_ptx_isa >= 880

} // namespace cuda_ptx

#endif // INSTRUCTIONS_TCGEN05_LD_H

// End of ./instructions/tcgen05_ld.h
// Start of ./instructions/tcgen05_mma.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_TCGEN05_MMA_H
#define INSTRUCTIONS_TCGEN05_MMA_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/tcgen05_mma.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// tcgen05.mma.cta_group.kind [d_tmem], a_desc, b_desc, idesc, disable_output_lane, enable_input_d, scale_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f
// .kind      = { .kind::f16, .kind::tf32 }
// .cta_group = { .cta_group::1 }
template <int N32, cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_1_t,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  const std::uint32_t (&disable_output_lane)[4],
  bool enable_input_d,
  cuda_ptx::n32_t<N32> scale_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_is_only_supported_on_SM_100a_100f_103a_103f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_1_t,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  const std::uint32_t (&disable_output_lane)[4],
  bool enable_input_d,
  cuda_ptx::n32_t<N32> scale_input_d)
{
  static_assert(kind == kind_f16 || kind == kind_tf32, "");
  // cta_group == cta_group_1 (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %8, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::f16 [%0], %1, %2, %3, {%4, %5, %6, %7}, PRED_enable_input_d, %9; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "n"(scale_input_d.value)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %8, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::tf32 [%0], %1, %2, %3, {%4, %5, %6, %7}, PRED_enable_input_d, %9; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "n"(scale_input_d.value)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_is_only_supported_on_SM_100a_100f_103a_103f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind [d_tmem], a_desc, b_desc, idesc, disable_output_lane, enable_input_d, scale_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f
// .kind      = { .kind::f16, .kind::tf32 }
// .cta_group = { .cta_group::2 }
template <int N32, cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_2_t,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  const std::uint32_t (&disable_output_lane)[8],
  bool enable_input_d,
  cuda_ptx::n32_t<N32> scale_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_is_only_supported_on_SM_100a_100f_103a_103f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_2_t,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  const std::uint32_t (&disable_output_lane)[8],
  bool enable_input_d,
  cuda_ptx::n32_t<N32> scale_input_d)
{
  static_assert(kind == kind_f16 || kind == kind_tf32, "");
  // cta_group == cta_group_2 (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %12, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::f16 [%0], %1, %2, %3, {%4, %5, %6, %7, %8, %9, %10, %11}, PRED_enable_input_d, %13; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(disable_output_lane[4]),
          "r"(disable_output_lane[5]),
          "r"(disable_output_lane[6]),
          "r"(disable_output_lane[7]),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "n"(scale_input_d.value)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %12, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::tf32 [%0], %1, %2, %3, {%4, %5, %6, %7, %8, %9, %10, %11}, PRED_enable_input_d, %13; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(disable_output_lane[4]),
          "r"(disable_output_lane[5]),
          "r"(disable_output_lane[6]),
          "r"(disable_output_lane[7]),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "n"(scale_input_d.value)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_is_only_supported_on_SM_100a_100f_103a_103f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind [d_tmem], a_desc, b_desc, idesc, disable_output_lane, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
// .cta_group = { .cta_group::1 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_1_t,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  const std::uint32_t (&disable_output_lane)[4],
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_1_t,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  const std::uint32_t (&disable_output_lane)[4],
  bool enable_input_d)
{
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  // cta_group == cta_group_1 (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %8, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::f16 [%0], %1, %2, %3, {%4, %5, %6, %7}, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %8, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::tf32 [%0], %1, %2, %3, {%4, %5, %6, %7}, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %8, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::f8f6f4 [%0], %1, %2, %3, {%4, %5, %6, %7}, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %8, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::i8 [%0], %1, %2, %3, {%4, %5, %6, %7}, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind [d_tmem], a_desc, b_desc, idesc, disable_output_lane, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
// .cta_group = { .cta_group::2 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_2_t,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  const std::uint32_t (&disable_output_lane)[8],
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_2_t,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  const std::uint32_t (&disable_output_lane)[8],
  bool enable_input_d)
{
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  // cta_group == cta_group_2 (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %12, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::f16 [%0], %1, %2, %3, {%4, %5, %6, %7, %8, %9, %10, %11}, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(disable_output_lane[4]),
          "r"(disable_output_lane[5]),
          "r"(disable_output_lane[6]),
          "r"(disable_output_lane[7]),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %12, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::tf32 [%0], %1, %2, %3, {%4, %5, %6, %7, %8, %9, %10, %11}, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(disable_output_lane[4]),
          "r"(disable_output_lane[5]),
          "r"(disable_output_lane[6]),
          "r"(disable_output_lane[7]),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %12, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::f8f6f4 [%0], %1, %2, %3, {%4, %5, %6, %7, %8, %9, %10, %11}, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(disable_output_lane[4]),
          "r"(disable_output_lane[5]),
          "r"(disable_output_lane[6]),
          "r"(disable_output_lane[7]),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %12, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::i8 [%0], %1, %2, %3, {%4, %5, %6, %7, %8, %9, %10, %11}, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(disable_output_lane[4]),
          "r"(disable_output_lane[5]),
          "r"(disable_output_lane[6]),
          "r"(disable_output_lane[7]),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind [d_tmem], a_desc, b_desc, idesc, enable_input_d, scale_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f
// .kind      = { .kind::f16, .kind::tf32 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <int N32, cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  cuda_ptx::n32_t<N32> scale_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_is_only_supported_on_SM_100a_100f_103a_103f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  cuda_ptx::n32_t<N32> scale_input_d)
{
  static_assert(kind == kind_f16 || kind == kind_tf32, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103)
    if constexpr (kind == kind_f16 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::f16 [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "n"(scale_input_d.value)
        : "memory"
      );
    } else if constexpr (kind == kind_f16 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::f16 [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "n"(scale_input_d.value)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::tf32 [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "n"(scale_input_d.value)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::tf32 [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "n"(scale_input_d.value)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_is_only_supported_on_SM_100a_100f_103a_103f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind [d_tmem], a_desc, b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::f16 [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f16 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::f16 [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::tf32 [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::tf32 [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::f8f6f4 [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::f8f6f4 [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::i8 [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_i8 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::i8 [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind [d_tmem], [a_tmem], b_desc, idesc, disable_output_lane, enable_input_d, scale_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f
// .kind      = { .kind::f16, .kind::tf32 }
// .cta_group = { .cta_group::1 }
template <int N32, cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_1_t,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  const std::uint32_t (&disable_output_lane)[4],
  bool enable_input_d,
  cuda_ptx::n32_t<N32> scale_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_1_t,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  const std::uint32_t (&disable_output_lane)[4],
  bool enable_input_d,
  cuda_ptx::n32_t<N32> scale_input_d)
{
  static_assert(kind == kind_f16 || kind == kind_tf32, "");
  // cta_group == cta_group_1 (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %8, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::f16 [%0], [%1], %2, %3, {%4, %5, %6, %7}, PRED_enable_input_d, %9; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "n"(scale_input_d.value)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %8, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::tf32 [%0], [%1], %2, %3, {%4, %5, %6, %7}, PRED_enable_input_d, %9; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "n"(scale_input_d.value)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind [d_tmem], [a_tmem], b_desc, idesc, disable_output_lane, enable_input_d, scale_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f
// .kind      = { .kind::f16, .kind::tf32 }
// .cta_group = { .cta_group::2 }
template <int N32, cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_2_t,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  const std::uint32_t (&disable_output_lane)[8],
  bool enable_input_d,
  cuda_ptx::n32_t<N32> scale_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_2_t,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  const std::uint32_t (&disable_output_lane)[8],
  bool enable_input_d,
  cuda_ptx::n32_t<N32> scale_input_d)
{
  static_assert(kind == kind_f16 || kind == kind_tf32, "");
  // cta_group == cta_group_2 (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %12, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::f16 [%0], [%1], %2, %3, {%4, %5, %6, %7, %8, %9, %10, %11}, PRED_enable_input_d, %13; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(disable_output_lane[4]),
          "r"(disable_output_lane[5]),
          "r"(disable_output_lane[6]),
          "r"(disable_output_lane[7]),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "n"(scale_input_d.value)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %12, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::tf32 [%0], [%1], %2, %3, {%4, %5, %6, %7, %8, %9, %10, %11}, PRED_enable_input_d, %13; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(disable_output_lane[4]),
          "r"(disable_output_lane[5]),
          "r"(disable_output_lane[6]),
          "r"(disable_output_lane[7]),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "n"(scale_input_d.value)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind [d_tmem], [a_tmem], b_desc, idesc, disable_output_lane, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
// .cta_group = { .cta_group::1 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_1_t,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  const std::uint32_t (&disable_output_lane)[4],
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_1_t,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  const std::uint32_t (&disable_output_lane)[4],
  bool enable_input_d)
{
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  // cta_group == cta_group_1 (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %8, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::f16 [%0], [%1], %2, %3, {%4, %5, %6, %7}, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %8, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::tf32 [%0], [%1], %2, %3, {%4, %5, %6, %7}, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %8, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::f8f6f4 [%0], [%1], %2, %3, {%4, %5, %6, %7}, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %8, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::i8 [%0], [%1], %2, %3, {%4, %5, %6, %7}, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind [d_tmem], [a_tmem], b_desc, idesc, disable_output_lane, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
// .cta_group = { .cta_group::2 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_2_t,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  const std::uint32_t (&disable_output_lane)[8],
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_2_t,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  const std::uint32_t (&disable_output_lane)[8],
  bool enable_input_d)
{
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  // cta_group == cta_group_2 (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %12, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::f16 [%0], [%1], %2, %3, {%4, %5, %6, %7, %8, %9, %10, %11}, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(disable_output_lane[4]),
          "r"(disable_output_lane[5]),
          "r"(disable_output_lane[6]),
          "r"(disable_output_lane[7]),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %12, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::tf32 [%0], [%1], %2, %3, {%4, %5, %6, %7, %8, %9, %10, %11}, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(disable_output_lane[4]),
          "r"(disable_output_lane[5]),
          "r"(disable_output_lane[6]),
          "r"(disable_output_lane[7]),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %12, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::f8f6f4 [%0], [%1], %2, %3, {%4, %5, %6, %7, %8, %9, %10, %11}, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(disable_output_lane[4]),
          "r"(disable_output_lane[5]),
          "r"(disable_output_lane[6]),
          "r"(disable_output_lane[7]),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %12, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::i8 [%0], [%1], %2, %3, {%4, %5, %6, %7, %8, %9, %10, %11}, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(disable_output_lane[0]),
          "r"(disable_output_lane[1]),
          "r"(disable_output_lane[2]),
          "r"(disable_output_lane[3]),
          "r"(disable_output_lane[4]),
          "r"(disable_output_lane[5]),
          "r"(disable_output_lane[6]),
          "r"(disable_output_lane[7]),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind [d_tmem], [a_tmem], b_desc, idesc, enable_input_d, scale_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f
// .kind      = { .kind::f16, .kind::tf32 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <int N32, cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  cuda_ptx::n32_t<N32> scale_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_depending_on_the_variant__();
template <int N32, cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  cuda_ptx::n32_t<N32> scale_input_d)
{
  static_assert(kind == kind_f16 || kind == kind_tf32, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103)
    if constexpr (kind == kind_f16 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::f16 [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "n"(scale_input_d.value)
        : "memory"
      );
    } else if constexpr (kind == kind_f16 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::f16 [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "n"(scale_input_d.value)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::tf32 [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "n"(scale_input_d.value)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::tf32 [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "n"(scale_input_d.value)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind [d_tmem], [a_tmem], b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::f16 [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f16 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::f16 [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::tf32 [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::tf32 [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::f8f6f4 [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::f8f6f4 [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::i8 [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_i8 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::i8 [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::1X [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf8f6f4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_1x_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf8f6f4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.scale_vec::1X [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.scale_vec::1X [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_1x_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::2X [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2x(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_2x_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2x(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.scale_vec::2X [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.scale_vec::2X [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::2X [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::2X [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_2x_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::4X [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_4x_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::4X [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::4X [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_4x_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.block16 [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block16_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block16 [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block16 [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block16_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.cta_group.kind.block_scale.block32 [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.block32 [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.block32 [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32 [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.block32 [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block32 [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block32 [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::1X [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf8f6f4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x_tmem_a(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_1x_tmem_a_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x_tmem_a(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf8f6f4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.scale_vec::1X [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.scale_vec::1X [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_1x_tmem_a_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::2X [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_2_tmem_a_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.scale_vec::2X [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.scale_vec::2X [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::2X [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::2X [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_2_tmem_a_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::4X [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x_tmem_a(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_4x_tmem_a_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x_tmem_a(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::4X [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::4X [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_4x_tmem_a_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.block16 [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16_tmem_a(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block16_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16_tmem_a(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block16 [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block16 [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block16_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.cta_group.kind.block_scale.block32 [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block32_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.block32 [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.block32 [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32 [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.block32 [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block32 [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block32 [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block32_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::1X.collector::a::fill [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf8f6f4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x_collector_a_fill(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_1x_collector_a_fill_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x_collector_a_fill(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf8f6f4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.scale_vec::1X.collector::a::fill [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.scale_vec::1X.collector::a::fill [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_1x_collector_a_fill_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::2X.collector::a::fill [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2x_collector_a_fill(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_2x_collector_a_fill_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2x_collector_a_fill(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.scale_vec::2X.collector::a::fill [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.scale_vec::2X.collector::a::fill [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::2X.collector::a::fill [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::2X.collector::a::fill [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_2x_collector_a_fill_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::4X.collector::a::fill [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x_collector_a_fill(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_4x_collector_a_fill_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x_collector_a_fill(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::4X.collector::a::fill [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::4X.collector::a::fill [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_4x_collector_a_fill_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.block16.collector::a::fill [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16_collector_a_fill(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block16_collector_a_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16_collector_a_fill(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::fill [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::fill [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block16_collector_a_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.cta_group.kind.block_scale.block32.collector::a::fill [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32_collector_a_fill(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block32_collector_a_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32_collector_a_fill(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.block32.collector::a::fill [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.block32.collector::a::fill [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32.collector::a::fill [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.block32.collector::a::fill [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::fill [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::fill [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block32_collector_a_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::1X.collector::a::fill [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf8f6f4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x_tmem_a_collector_a_fill(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_1x_tmem_a_collector_a_fill_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x_tmem_a_collector_a_fill(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf8f6f4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.scale_vec::1X.collector::a::fill [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.scale_vec::1X.collector::a::fill [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_1x_tmem_a_collector_a_fill_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::2X.collector::a::fill [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2_tmem_a_collector_a_fill(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_2_tmem_a_collector_a_fill_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2_tmem_a_collector_a_fill(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.scale_vec::2X.collector::a::fill [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.scale_vec::2X.collector::a::fill [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::2X.collector::a::fill [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::2X.collector::a::fill [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_2_tmem_a_collector_a_fill_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::4X.collector::a::fill [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x_tmem_a_collector_a_fill(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_4x_tmem_a_collector_a_fill_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x_tmem_a_collector_a_fill(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::4X.collector::a::fill [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::4X.collector::a::fill [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_4x_tmem_a_collector_a_fill_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.block16.collector::a::fill [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16_tmem_a_collector_a_fill(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block16_tmem_a_collector_a_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16_tmem_a_collector_a_fill(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::fill [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::fill [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block16_tmem_a_collector_a_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.cta_group.kind.block_scale.block32.collector::a::fill [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32_tmem_a_collector_a_fill(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block32_tmem_a_collector_a_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32_tmem_a_collector_a_fill(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.block32.collector::a::fill [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.block32.collector::a::fill [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32.collector::a::fill [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.block32.collector::a::fill [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::fill [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::fill [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block32_tmem_a_collector_a_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::1X.collector::a::use [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf8f6f4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x_collector_a_use(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_1x_collector_a_use_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x_collector_a_use(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf8f6f4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.scale_vec::1X.collector::a::use [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.scale_vec::1X.collector::a::use [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_1x_collector_a_use_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::2X.collector::a::use [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2x_collector_a_use(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_2x_collector_a_use_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2x_collector_a_use(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.scale_vec::2X.collector::a::use [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.scale_vec::2X.collector::a::use [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::2X.collector::a::use [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::2X.collector::a::use [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_2x_collector_a_use_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::4X.collector::a::use [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x_collector_a_use(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_4x_collector_a_use_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x_collector_a_use(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::4X.collector::a::use [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::4X.collector::a::use [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_4x_collector_a_use_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.block16.collector::a::use [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16_collector_a_use(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block16_collector_a_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16_collector_a_use(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::use [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::use [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block16_collector_a_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.cta_group.kind.block_scale.block32.collector::a::use [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32_collector_a_use(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block32_collector_a_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32_collector_a_use(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.block32.collector::a::use [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.block32.collector::a::use [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32.collector::a::use [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.block32.collector::a::use [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::use [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::use [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block32_collector_a_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::1X.collector::a::use [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf8f6f4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x_tmem_a_collector_a_use(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_1x_tmem_a_collector_a_use_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x_tmem_a_collector_a_use(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf8f6f4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.scale_vec::1X.collector::a::use [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.scale_vec::1X.collector::a::use [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_1x_tmem_a_collector_a_use_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::2X.collector::a::use [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2_tmem_a_collector_a_use(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_2_tmem_a_collector_a_use_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2_tmem_a_collector_a_use(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.scale_vec::2X.collector::a::use [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.scale_vec::2X.collector::a::use [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::2X.collector::a::use [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::2X.collector::a::use [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_2_tmem_a_collector_a_use_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::4X.collector::a::use [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x_tmem_a_collector_a_use(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_4x_tmem_a_collector_a_use_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x_tmem_a_collector_a_use(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::4X.collector::a::use [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::4X.collector::a::use [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_4x_tmem_a_collector_a_use_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.block16.collector::a::use [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16_tmem_a_collector_a_use(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block16_tmem_a_collector_a_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16_tmem_a_collector_a_use(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::use [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::use [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block16_tmem_a_collector_a_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.cta_group.kind.block_scale.block32.collector::a::use [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32_tmem_a_collector_a_use(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block32_tmem_a_collector_a_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32_tmem_a_collector_a_use(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.block32.collector::a::use [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.block32.collector::a::use [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32.collector::a::use [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.block32.collector::a::use [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::use [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::use [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block32_tmem_a_collector_a_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::1X.collector::a::lastuse [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf8f6f4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x_collector_a_lastuse(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_1x_collector_a_lastuse_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x_collector_a_lastuse(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf8f6f4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.scale_vec::1X.collector::a::lastuse [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.scale_vec::1X.collector::a::lastuse [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_1x_collector_a_lastuse_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::2X.collector::a::lastuse [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2x_collector_a_lastuse(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_2x_collector_a_lastuse_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2x_collector_a_lastuse(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.scale_vec::2X.collector::a::lastuse [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.scale_vec::2X.collector::a::lastuse [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::2X.collector::a::lastuse [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::2X.collector::a::lastuse [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_2x_collector_a_lastuse_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::4X.collector::a::lastuse [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x_collector_a_lastuse(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_4x_collector_a_lastuse_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x_collector_a_lastuse(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::4X.collector::a::lastuse [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::4X.collector::a::lastuse [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_4x_collector_a_lastuse_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.block16.collector::a::lastuse [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16_collector_a_lastuse(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block16_collector_a_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16_collector_a_lastuse(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block16_collector_a_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.cta_group.kind.block_scale.block32.collector::a::lastuse [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32_collector_a_lastuse(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block32_collector_a_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32_collector_a_lastuse(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.block32.collector::a::lastuse [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.block32.collector::a::lastuse [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32.collector::a::lastuse [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.block32.collector::a::lastuse [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block32_collector_a_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::1X.collector::a::lastuse [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf8f6f4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x_tmem_a_collector_a_lastuse(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_1x_tmem_a_collector_a_lastuse_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x_tmem_a_collector_a_lastuse(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf8f6f4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.scale_vec::1X.collector::a::lastuse [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.scale_vec::1X.collector::a::lastuse [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_1x_tmem_a_collector_a_lastuse_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::2X.collector::a::lastuse [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2_tmem_a_collector_a_lastuse(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_2_tmem_a_collector_a_lastuse_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2_tmem_a_collector_a_lastuse(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.scale_vec::2X.collector::a::lastuse [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.scale_vec::2X.collector::a::lastuse [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::2X.collector::a::lastuse [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::2X.collector::a::lastuse [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_2_tmem_a_collector_a_lastuse_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::4X.collector::a::lastuse [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x_tmem_a_collector_a_lastuse(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_4x_tmem_a_collector_a_lastuse_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x_tmem_a_collector_a_lastuse(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::4X.collector::a::lastuse [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::4X.collector::a::lastuse [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_4x_tmem_a_collector_a_lastuse_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.block16.collector::a::lastuse [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16_tmem_a_collector_a_lastuse(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block16_tmem_a_collector_a_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16_tmem_a_collector_a_lastuse(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block16_tmem_a_collector_a_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.cta_group.kind.block_scale.block32.collector::a::lastuse [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32_tmem_a_collector_a_lastuse(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block32_tmem_a_collector_a_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32_tmem_a_collector_a_lastuse(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.block32.collector::a::lastuse [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.block32.collector::a::lastuse [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32.collector::a::lastuse [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.block32.collector::a::lastuse [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block32_tmem_a_collector_a_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::1X.collector::a::discard [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf8f6f4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x_collector_a_discard(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_1x_collector_a_discard_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x_collector_a_discard(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf8f6f4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.scale_vec::1X.collector::a::discard [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.scale_vec::1X.collector::a::discard [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_1x_collector_a_discard_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::2X.collector::a::discard [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2x_collector_a_discard(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_2x_collector_a_discard_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2x_collector_a_discard(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.scale_vec::2X.collector::a::discard [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.scale_vec::2X.collector::a::discard [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::2X.collector::a::discard [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::2X.collector::a::discard [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_2x_collector_a_discard_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::4X.collector::a::discard [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x_collector_a_discard(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_4x_collector_a_discard_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x_collector_a_discard(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::4X.collector::a::discard [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::4X.collector::a::discard [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_4x_collector_a_discard_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.block16.collector::a::discard [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16_collector_a_discard(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block16_collector_a_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16_collector_a_discard(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::discard [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::discard [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block16_collector_a_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.cta_group.kind.block_scale.block32.collector::a::discard [d_tmem], a_desc, b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32_collector_a_discard(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block32_collector_a_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32_collector_a_discard(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.block32.collector::a::discard [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.block32.collector::a::discard [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32.collector::a::discard [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.block32.collector::a::discard [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::discard [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::discard [%0], %1, %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block32_collector_a_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::1X.collector::a::discard [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf8f6f4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x_tmem_a_collector_a_discard(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_1x_tmem_a_collector_a_discard_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_1x_tmem_a_collector_a_discard(
  cuda_ptx::kind_mxf8f6f4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf8f6f4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.scale_vec::1X.collector::a::discard [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.scale_vec::1X.collector::a::discard [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_1x_tmem_a_collector_a_discard_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::2X.collector::a::discard [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2_tmem_a_collector_a_discard(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_2_tmem_a_collector_a_discard_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_2_tmem_a_collector_a_discard(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.scale_vec::2X.collector::a::discard [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.scale_vec::2X.collector::a::discard [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::2X.collector::a::discard [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::2X.collector::a::discard [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_2_tmem_a_collector_a_discard_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.scale_vec::4X.collector::a::discard [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x_tmem_a_collector_a_discard(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_vec_4x_tmem_a_collector_a_discard_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_vec_4x_tmem_a_collector_a_discard(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.scale_vec::4X.collector::a::discard [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.scale_vec::4X.collector::a::discard [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_vec_4x_tmem_a_collector_a_discard_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.cta_group.kind.block_scale.block16.collector::a::discard [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16_tmem_a_collector_a_discard(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block16_tmem_a_collector_a_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block16_tmem_a_collector_a_discard(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::discard [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::discard [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block16_tmem_a_collector_a_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.cta_group.kind.block_scale.block32.collector::a::discard [d_tmem], [a_tmem], b_desc, idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32_tmem_a_collector_a_discard(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_block_scale_block32_tmem_a_collector_a_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_block_scale_block32_tmem_a_collector_a_discard(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf8f6f4.block_scale.block32.collector::a::discard [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale.block32.collector::a::discard [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4.block_scale.block32.collector::a::discard [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4.block_scale.block32.collector::a::discard [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::discard [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %6, 0; \n\t"
        "tcgen05.mma.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::discard [%0], [%1], %2, %3, [%4], [%5], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_block_scale_block32_tmem_a_collector_a_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

} // namespace cuda_ptx

#endif // INSTRUCTIONS_TCGEN05_MMA_H

// End of ./instructions/tcgen05_mma.h
// Start of ./instructions/tcgen05_mma_sp.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_TCGEN05_MMA_SP_H
#define INSTRUCTIONS_TCGEN05_MMA_SP_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/tcgen05_mma_sp.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// tcgen05.mma.sp.cta_group.kind [d_tmem], a_desc, b_desc, [sp_info_tmem], idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::f16, .kind::f8f6f4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  bool enable_input_d)
{
  static_assert(kind == kind_f16 || kind == kind_f8f6f4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %5, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::f16 [%0], %1, %2, [%3], %4, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f16 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %5, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::f16 [%0], %1, %2, [%3], %4, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %5, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::f8f6f4 [%0], %1, %2, [%3], %4, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %5, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::f8f6f4 [%0], %1, %2, [%3], %4, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.sp.cta_group.kind [d_tmem], [a_tmem], b_desc, [sp_info_tmem], idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::f16, .kind::f8f6f4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  bool enable_input_d)
{
  static_assert(kind == kind_f16 || kind == kind_f8f6f4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %5, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::f16 [%0], [%1], %2, [%3], %4, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f16 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %5, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::f16 [%0], [%1], %2, [%3], %4, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %5, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::f8f6f4 [%0], [%1], %2, [%3], %4, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %5, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::f8f6f4 [%0], [%1], %2, [%3], %4, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block16 [d_tmem], a_desc, b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block16_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16 [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16 [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block16_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block32 [d_tmem], a_desc, b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32 [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32 [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32 [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32 [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf8f6f4.block_scale.block32 [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf8f6f4.block_scale.block32 [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block32_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block16 [d_tmem], [a_tmem], b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16_tmem_a(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block16_tmem_a_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16_tmem_a(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16 [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16 [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block16_tmem_a_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block32 [d_tmem], [a_tmem], b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block32_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32_tmem_a(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32 [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32 [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32 [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32 [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf8f6f4.block_scale.block32 [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf8f6f4.block_scale.block32 [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block32_tmem_a_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block16.collector::a::fill [d_tmem], a_desc, b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16_collector_a_fill(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block16_collector_a_fill_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16_collector_a_fill(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::fill [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::fill [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block16_collector_a_fill_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block32.collector::a::fill [d_tmem], a_desc, b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32_collector_a_fill(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block32_collector_a_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32_collector_a_fill(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::fill [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::fill [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::fill [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::fill [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf8f6f4.block_scale.block32.collector::a::fill [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf8f6f4.block_scale.block32.collector::a::fill [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block32_collector_a_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block16.collector::a::fill [d_tmem], [a_tmem], b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16_tmem_a_collector_a_fill(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block16_tmem_a_collector_a_fill_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16_tmem_a_collector_a_fill(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::fill [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::fill [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block16_tmem_a_collector_a_fill_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block32.collector::a::fill [d_tmem], [a_tmem], b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32_tmem_a_collector_a_fill(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block32_tmem_a_collector_a_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32_tmem_a_collector_a_fill(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::fill [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::fill [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::fill [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::fill [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf8f6f4.block_scale.block32.collector::a::fill [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf8f6f4.block_scale.block32.collector::a::fill [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block32_tmem_a_collector_a_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block16.collector::a::use [d_tmem], a_desc, b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16_collector_a_use(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block16_collector_a_use_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16_collector_a_use(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::use [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::use [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block16_collector_a_use_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block32.collector::a::use [d_tmem], a_desc, b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32_collector_a_use(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block32_collector_a_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32_collector_a_use(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::use [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::use [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::use [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::use [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf8f6f4.block_scale.block32.collector::a::use [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf8f6f4.block_scale.block32.collector::a::use [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block32_collector_a_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block16.collector::a::use [d_tmem], [a_tmem], b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16_tmem_a_collector_a_use(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block16_tmem_a_collector_a_use_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16_tmem_a_collector_a_use(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::use [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::use [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block16_tmem_a_collector_a_use_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block32.collector::a::use [d_tmem], [a_tmem], b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32_tmem_a_collector_a_use(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block32_tmem_a_collector_a_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32_tmem_a_collector_a_use(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::use [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::use [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::use [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::use [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf8f6f4.block_scale.block32.collector::a::use [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf8f6f4.block_scale.block32.collector::a::use [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block32_tmem_a_collector_a_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block16.collector::a::lastuse [d_tmem], a_desc, b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16_collector_a_lastuse(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block16_collector_a_lastuse_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16_collector_a_lastuse(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block16_collector_a_lastuse_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block32.collector::a::lastuse [d_tmem], a_desc, b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32_collector_a_lastuse(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block32_collector_a_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32_collector_a_lastuse(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::lastuse [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::lastuse [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf8f6f4.block_scale.block32.collector::a::lastuse [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf8f6f4.block_scale.block32.collector::a::lastuse [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block32_collector_a_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block16.collector::a::lastuse [d_tmem], [a_tmem], b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16_tmem_a_collector_a_lastuse(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block16_tmem_a_collector_a_lastuse_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16_tmem_a_collector_a_lastuse(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::lastuse [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block16_tmem_a_collector_a_lastuse_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block32.collector::a::lastuse [d_tmem], [a_tmem], b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32_tmem_a_collector_a_lastuse(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block32_tmem_a_collector_a_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32_tmem_a_collector_a_lastuse(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::lastuse [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::lastuse [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::lastuse [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf8f6f4.block_scale.block32.collector::a::lastuse [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf8f6f4.block_scale.block32.collector::a::lastuse [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block32_tmem_a_collector_a_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block16.collector::a::discard [d_tmem], a_desc, b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16_collector_a_discard(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block16_collector_a_discard_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16_collector_a_discard(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::discard [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::discard [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block16_collector_a_discard_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block32.collector::a::discard [d_tmem], a_desc, b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32_collector_a_discard(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block32_collector_a_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32_collector_a_discard(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::discard [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::discard [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::discard [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::discard [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf8f6f4.block_scale.block32.collector::a::discard [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf8f6f4.block_scale.block32.collector::a::discard [%0], %1, %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block32_collector_a_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block16.collector::a::discard [d_tmem], [a_tmem], b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_103a, SM_110a
// .kind      = { .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16_tmem_a_collector_a_discard(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block16_tmem_a_collector_a_discard_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block16_tmem_a_collector_a_discard(
  cuda_ptx::kind_mxf4nvf4_t,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  // kind == kind_mxf4nvf4 (due to parameter type constraint)
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block16.collector::a::discard [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block16.collector::a::discard [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block16_tmem_a_collector_a_discard_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

/*
// tcgen05.mma.sp.cta_group.kind.block_scale.block32.collector::a::discard [d_tmem], [a_tmem], b_desc, [sp_info_tmem], idesc, [scale_A_tmem], [scale_B_tmem], enable_input_d; // PTX ISA 88, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .kind      = { .kind::mxf8f6f4, .kind::mxf4, .kind::mxf4nvf4 }
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32_tmem_a_collector_a_discard(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 880
extern "C" __device__ void __cuda_ptx_tcgen05_mma_sp_block_scale_block32_tmem_a_collector_a_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind, cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_mma_sp_block_scale_block32_tmem_a_collector_a_discard(
  cuda_ptx::kind_t<Kind> kind,
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t sp_info_tmem,
  std::uint32_t idesc,
  std::uint32_t scale_A_tmem,
  std::uint32_t scale_B_tmem,
  bool enable_input_d)
{
  static_assert(kind == kind_mxf8f6f4 || kind == kind_mxf4 || kind == kind_mxf4nvf4, "");
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_mxf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4.block_scale.block32.collector::a::discard [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4.block_scale.block32.collector::a::discard [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf4nvf4.block_scale.block32.collector::a::discard [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf4nvf4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf4nvf4.block_scale.block32.collector::a::discard [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_1) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::1.kind::mxf8f6f4.block_scale.block32.collector::a::discard [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_mxf8f6f4 && cta_group == cta_group_2) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %7, 0; \n\t"
        "tcgen05.mma.sp.cta_group::2.kind::mxf8f6f4.block_scale.block32.collector::a::discard [%0], [%1], %2, [%3], %4, [%5], [%6], PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(sp_info_tmem),
          "r"(idesc),
          "r"(scale_A_tmem),
          "r"(scale_B_tmem),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_sp_block_scale_block32_tmem_a_collector_a_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 880

} // namespace cuda_ptx

#endif // INSTRUCTIONS_TCGEN05_MMA_SP_H

// End of ./instructions/tcgen05_mma_sp.h
// Start of ./instructions/tcgen05_mma_ws.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_TCGEN05_MMA_WS_H
#define INSTRUCTIONS_TCGEN05_MMA_WS_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/tcgen05_mma_ws.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// tcgen05.mma.ws.cta_group.kind.collector::b0::fill [d_tmem], a_desc, b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b0_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b0_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b0_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b0::fill [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b0::fill [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b0::fill [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b0::fill [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b0_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b0::fill [d_tmem], a_desc, b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b0_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b0_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b0_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b0::fill [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b0::fill [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b0::fill [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b0::fill [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b0_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b0::fill [d_tmem], [a_tmem], b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b0_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b0_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b0_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b0::fill [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b0::fill [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b0::fill [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b0::fill [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b0_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b0::fill [d_tmem], [a_tmem], b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b0_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b0_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b0_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b0::fill [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b0::fill [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b0::fill [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b0::fill [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b0_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b0::use [d_tmem], a_desc, b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b0_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b0_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b0_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b0::use [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b0::use [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b0::use [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b0::use [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b0_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b0::use [d_tmem], a_desc, b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b0_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b0_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b0_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b0::use [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b0::use [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b0::use [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b0::use [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b0_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b0::use [d_tmem], [a_tmem], b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b0_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b0_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b0_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b0::use [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b0::use [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b0::use [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b0::use [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b0_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b0::use [d_tmem], [a_tmem], b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b0_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b0_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b0_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b0::use [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b0::use [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b0::use [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b0::use [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b0_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b0::lastuse [d_tmem], a_desc, b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b0_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b0_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b0_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b0::lastuse [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b0::lastuse [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b0::lastuse [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b0::lastuse [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b0_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b0::lastuse [d_tmem], a_desc, b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b0_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b0_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b0_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b0::lastuse [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b0::lastuse [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b0::lastuse [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b0::lastuse [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b0_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b0::lastuse [d_tmem], [a_tmem], b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b0_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b0_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b0_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b0::lastuse [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b0::lastuse [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b0::lastuse [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b0::lastuse [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b0_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b0::lastuse [d_tmem], [a_tmem], b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b0_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b0_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b0_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b0::lastuse [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b0::lastuse [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b0::lastuse [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b0::lastuse [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b0_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b0::discard [d_tmem], a_desc, b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b0_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b0_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b0_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b0::discard [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b0::discard [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b0::discard [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b0::discard [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b0_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b0::discard [d_tmem], a_desc, b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b0_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b0_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b0_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b0::discard [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b0::discard [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b0::discard [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b0::discard [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b0_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b0::discard [d_tmem], [a_tmem], b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b0_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b0_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b0_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b0::discard [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b0::discard [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b0::discard [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b0::discard [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b0_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b0::discard [d_tmem], [a_tmem], b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b0_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b0_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b0_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b0::discard [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b0::discard [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b0::discard [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b0::discard [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b0_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b1::fill [d_tmem], a_desc, b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b1_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b1_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b1_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b1::fill [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b1::fill [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b1::fill [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b1::fill [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b1_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b1::fill [d_tmem], a_desc, b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b1_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b1_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b1_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b1::fill [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b1::fill [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b1::fill [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b1::fill [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b1_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b1::fill [d_tmem], [a_tmem], b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b1_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b1_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b1_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b1::fill [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b1::fill [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b1::fill [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b1::fill [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b1_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b1::fill [d_tmem], [a_tmem], b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b1_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b1_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b1_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b1::fill [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b1::fill [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b1::fill [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b1::fill [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b1_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b1::use [d_tmem], a_desc, b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b1_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b1_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b1_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b1::use [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b1::use [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b1::use [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b1::use [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b1_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b1::use [d_tmem], a_desc, b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b1_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b1_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b1_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b1::use [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b1::use [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b1::use [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b1::use [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b1_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b1::use [d_tmem], [a_tmem], b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b1_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b1_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b1_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b1::use [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b1::use [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b1::use [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b1::use [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b1_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b1::use [d_tmem], [a_tmem], b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b1_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b1_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b1_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b1::use [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b1::use [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b1::use [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b1::use [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b1_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b1::lastuse [d_tmem], a_desc, b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b1_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b1_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b1_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b1::lastuse [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b1::lastuse [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b1::lastuse [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b1::lastuse [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b1_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b1::lastuse [d_tmem], a_desc, b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b1_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b1_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b1_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b1::lastuse [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b1::lastuse [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b1::lastuse [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b1::lastuse [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b1_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b1::lastuse [d_tmem], [a_tmem], b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b1_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b1_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b1_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b1::lastuse [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b1::lastuse [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b1::lastuse [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b1::lastuse [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b1_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b1::lastuse [d_tmem], [a_tmem], b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b1_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b1_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b1_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b1::lastuse [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b1::lastuse [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b1::lastuse [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b1::lastuse [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b1_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b1::discard [d_tmem], a_desc, b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b1_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b1_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b1_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b1::discard [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b1::discard [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b1::discard [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b1::discard [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b1_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b1::discard [d_tmem], a_desc, b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b1_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b1_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b1_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b1::discard [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b1::discard [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b1::discard [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b1::discard [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b1_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b1::discard [d_tmem], [a_tmem], b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b1_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b1_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b1_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b1::discard [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b1::discard [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b1::discard [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b1::discard [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b1_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b1::discard [d_tmem], [a_tmem], b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b1_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b1_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b1_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b1::discard [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b1::discard [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b1::discard [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b1::discard [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b1_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b2::fill [d_tmem], a_desc, b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b2_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b2_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b2_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b2::fill [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b2::fill [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b2::fill [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b2::fill [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b2_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b2::fill [d_tmem], a_desc, b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b2_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b2_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b2_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b2::fill [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b2::fill [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b2::fill [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b2::fill [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b2_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b2::fill [d_tmem], [a_tmem], b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b2_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b2_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b2_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b2::fill [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b2::fill [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b2::fill [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b2::fill [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b2_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b2::fill [d_tmem], [a_tmem], b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b2_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b2_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b2_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b2::fill [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b2::fill [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b2::fill [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b2::fill [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b2_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b2::use [d_tmem], a_desc, b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b2_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b2_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b2_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b2::use [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b2::use [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b2::use [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b2::use [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b2_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b2::use [d_tmem], a_desc, b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b2_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b2_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b2_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b2::use [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b2::use [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b2::use [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b2::use [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b2_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b2::use [d_tmem], [a_tmem], b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b2_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b2_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b2_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b2::use [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b2::use [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b2::use [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b2::use [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b2_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b2::use [d_tmem], [a_tmem], b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b2_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b2_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b2_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b2::use [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b2::use [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b2::use [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b2::use [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b2_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b2::lastuse [d_tmem], a_desc, b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b2_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b2_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b2_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b2::lastuse [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b2::lastuse [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b2::lastuse [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b2::lastuse [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b2_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b2::lastuse [d_tmem], a_desc, b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b2_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b2_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b2_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b2::lastuse [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b2::lastuse [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b2::lastuse [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b2::lastuse [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b2_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b2::lastuse [d_tmem], [a_tmem], b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b2_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b2_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b2_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b2::lastuse [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b2::lastuse [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b2::lastuse [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b2::lastuse [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b2_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b2::lastuse [d_tmem], [a_tmem], b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b2_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b2_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b2_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b2::lastuse [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b2::lastuse [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b2::lastuse [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b2::lastuse [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b2_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b2::discard [d_tmem], a_desc, b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b2_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b2_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b2_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b2::discard [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b2::discard [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b2::discard [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b2::discard [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b2_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b2::discard [d_tmem], a_desc, b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b2_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b2_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b2_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b2::discard [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b2::discard [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b2::discard [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b2::discard [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b2_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b2::discard [d_tmem], [a_tmem], b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b2_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b2_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b2_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b2::discard [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b2::discard [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b2::discard [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b2::discard [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b2_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b2::discard [d_tmem], [a_tmem], b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b2_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b2_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b2_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b2::discard [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b2::discard [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b2::discard [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b2::discard [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b2_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b3::fill [d_tmem], a_desc, b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b3_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b3_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b3_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b3::fill [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b3::fill [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b3::fill [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b3::fill [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b3_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b3::fill [d_tmem], a_desc, b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b3_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b3_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b3_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b3::fill [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b3::fill [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b3::fill [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b3::fill [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b3_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b3::fill [d_tmem], [a_tmem], b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b3_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b3_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b3_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b3::fill [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b3::fill [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b3::fill [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b3::fill [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b3_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b3::fill [d_tmem], [a_tmem], b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b3_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b3_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b3_fill(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b3::fill [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b3::fill [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b3::fill [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b3::fill [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b3_fill_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b3::use [d_tmem], a_desc, b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b3_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b3_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b3_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b3::use [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b3::use [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b3::use [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b3::use [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b3_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b3::use [d_tmem], a_desc, b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b3_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b3_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b3_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b3::use [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b3::use [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b3::use [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b3::use [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b3_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b3::use [d_tmem], [a_tmem], b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b3_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b3_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b3_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b3::use [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b3::use [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b3::use [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b3::use [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b3_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b3::use [d_tmem], [a_tmem], b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b3_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b3_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b3_use(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b3::use [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b3::use [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b3::use [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b3::use [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b3_use_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b3::lastuse [d_tmem], a_desc, b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b3_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b3_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b3_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b3::lastuse [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b3::lastuse [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b3::lastuse [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b3::lastuse [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b3_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b3::lastuse [d_tmem], a_desc, b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b3_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b3_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b3_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b3::lastuse [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b3::lastuse [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b3::lastuse [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b3::lastuse [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b3_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b3::lastuse [d_tmem], [a_tmem], b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b3_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b3_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b3_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b3::lastuse [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b3::lastuse [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b3::lastuse [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b3::lastuse [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b3_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b3::lastuse [d_tmem], [a_tmem], b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b3_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b3_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b3_lastuse(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b3::lastuse [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b3::lastuse [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b3::lastuse [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b3::lastuse [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b3_lastuse_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b3::discard [d_tmem], a_desc, b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b3_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b3_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b3_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b3::discard [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b3::discard [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b3::discard [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b3::discard [%0], %1, %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b3_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b3::discard [d_tmem], a_desc, b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b3_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_collector_b3_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_collector_b3_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint64_t a_desc,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b3::discard [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b3::discard [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b3::discard [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b3::discard [%0], %1, %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "l"(a_desc),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_collector_b3_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b3::discard [d_tmem], [a_tmem], b_desc, idesc, enable_input_d, zero_column_mask_desc; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b3_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b3_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b3_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d,
  std::uint64_t zero_column_mask_desc)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b3::discard [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b3::discard [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b3::discard [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b3::discard [%0], [%1], %2, %3, PRED_enable_input_d, %5; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d)),
          "l"(zero_column_mask_desc)
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b3_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.mma.ws.cta_group.kind.collector::b3::discard [d_tmem], [a_tmem], b_desc, idesc, enable_input_d; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
// .cta_group = { .cta_group::1 }
// .kind      = { .kind::f16, .kind::tf32, .kind::f8f6f4, .kind::i8 }
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b3_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b3_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
template <cuda_ptx::dot_kind Kind>
__device__ static inline void tcgen05_mma_ws_tmem_a_collector_b3_discard(
  cuda_ptx::cta_group_1_t,
  cuda_ptx::kind_t<Kind> kind,
  std::uint32_t d_tmem,
  std::uint32_t a_tmem,
  std::uint64_t b_desc,
  std::uint32_t idesc,
  bool enable_input_d)
{
  // cta_group == cta_group_1 (due to parameter type constraint)
  static_assert(kind == kind_f16 || kind == kind_tf32 || kind == kind_f8f6f4 || kind == kind_i8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    if constexpr (kind == kind_f16) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f16.collector::b3::discard [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_tf32) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::tf32.collector::b3::discard [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    } else if constexpr (kind == kind_f8f6f4) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::f8f6f4.collector::b3::discard [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }
  #endif
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (kind == kind_i8) {
      asm volatile (
        "{\n\t"
        ".reg .pred PRED_enable_input_d; \n\t"
        "setp.ne.b32 PRED_enable_input_d, %4, 0; \n\t"
        "tcgen05.mma.ws.cta_group::1.kind::i8.collector::b3::discard [%0], [%1], %2, %3, PRED_enable_input_d; \n"
        "}"
        :
        : "r"(d_tmem),
          "r"(a_tmem),
          "l"(b_desc),
          "r"(idesc),
          "r"(static_cast<std::uint32_t>(enable_input_d))
        : "memory"
      );
    }

  #endif
  #if !((defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)) || (defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)))
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_mma_ws_tmem_a_collector_b3_discard_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

} // namespace cuda_ptx

#endif // INSTRUCTIONS_TCGEN05_MMA_WS_H

// End of ./instructions/tcgen05_mma_ws.h
// Start of ./instructions/tcgen05_shift.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_TCGEN05_SHIFT_H
#define INSTRUCTIONS_TCGEN05_SHIFT_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/tcgen05_shift.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// tcgen05.shift.cta_group.down [taddr]; // PTX ISA 86, SM_100a, SM_103a, SM_110a
// .cta_group = { .cta_group::1, .cta_group::2 }
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_shift_down(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_shift_down_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
template <cuda_ptx::dot_cta_group Cta_Group>
__device__ static inline void tcgen05_shift_down(
  cuda_ptx::cta_group_t<Cta_Group> cta_group,
  std::uint32_t taddr)
{
  static_assert(cta_group == cta_group_1 || cta_group == cta_group_2, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100)
    if constexpr (cta_group == cta_group_1) {
      asm volatile (
        "tcgen05.shift.cta_group::1.down [%0];"
        :
        : "r"(taddr)
        : "memory"
      );
    } else if constexpr (cta_group == cta_group_2) {
      asm volatile (
        "tcgen05.shift.cta_group::2.down [%0];"
        :
        : "r"(taddr)
        : "memory"
      );
    }

  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_shift_down_is_only_supported_on_SM_100a_103a_110a_depending_on_the_variant__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

} // namespace cuda_ptx

#endif // INSTRUCTIONS_TCGEN05_SHIFT_H

// End of ./instructions/tcgen05_shift.h
// Start of ./instructions/tcgen05_st.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_TCGEN05_ST_H
#define INSTRUCTIONS_TCGEN05_ST_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/tcgen05_st.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// tcgen05.st.sync.aligned.16x64b.x1.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b(
  std::uint32_t taddr,
  const B32 (&values)[1]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b(
  std::uint32_t taddr,
  const B32 (&values)[1])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x64b.x1.b32 [%0], {%1};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x64b.x1.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[1]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x64b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[1])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x64b.x1.unpack::16b.b32 [%0], {%1};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x64b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x64b.x2.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b(
  std::uint32_t taddr,
  const B32 (&values)[2]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b(
  std::uint32_t taddr,
  const B32 (&values)[2])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x64b.x2.b32 [%0], {%1, %2};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x64b.x2.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[2]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x64b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[2])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x64b.x2.unpack::16b.b32 [%0], {%1, %2};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x64b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x64b.x4.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b(
  std::uint32_t taddr,
  const B32 (&values)[4]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b(
  std::uint32_t taddr,
  const B32 (&values)[4])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x64b.x4.b32 [%0], {%1, %2, %3, %4};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x64b.x4.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[4]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x64b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[4])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x64b.x4.unpack::16b.b32 [%0], {%1, %2, %3, %4};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x64b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x64b.x8.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b(
  std::uint32_t taddr,
  const B32 (&values)[8]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b(
  std::uint32_t taddr,
  const B32 (&values)[8])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x64b.x8.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x64b.x8.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[8]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x64b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[8])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x64b.x8.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x64b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x64b.x16.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b(
  std::uint32_t taddr,
  const B32 (&values)[16]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b(
  std::uint32_t taddr,
  const B32 (&values)[16])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x64b.x16.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x64b.x16.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[16]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x64b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[16])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x64b.x16.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x64b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x64b.x32.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b(
  std::uint32_t taddr,
  const B32 (&values)[32]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b(
  std::uint32_t taddr,
  const B32 (&values)[32])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x64b.x32.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x64b.x32.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[32]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x64b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[32])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x64b.x32.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x64b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x64b.x64.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b(
  std::uint32_t taddr,
  const B32 (&values)[64]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b(
  std::uint32_t taddr,
  const B32 (&values)[64])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x64b.x64.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x64b.x64.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[64]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x64b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[64])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x64b.x64.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x64b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x64b.x128.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b(
  std::uint32_t taddr,
  const B32 (&values)[128]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b(
  std::uint32_t taddr,
  const B32 (&values)[128])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x64b.x128.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127, %128};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[64])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[65])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[66])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[67])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[68])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[69])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[70])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[71])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[72])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[73])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[74])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[75])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[76])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[77])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[78])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[79])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[80])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[81])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[82])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[83])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[84])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[85])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[86])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[87])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[88])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[89])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[90])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[91])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[92])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[93])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[94])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[95])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[96])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[97])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[98])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[99])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[100])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[101])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[102])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[103])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[104])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[105])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[106])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[107])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[108])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[109])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[110])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[111])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[112])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[113])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[114])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[115])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[116])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[117])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[118])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[119])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[120])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[121])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[122])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[123])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[124])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[125])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[126])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[127]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x64b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x64b.x128.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[128]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x64b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x64b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[128])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x64b.x128.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127, %128};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[64])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[65])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[66])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[67])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[68])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[69])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[70])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[71])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[72])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[73])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[74])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[75])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[76])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[77])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[78])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[79])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[80])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[81])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[82])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[83])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[84])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[85])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[86])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[87])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[88])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[89])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[90])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[91])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[92])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[93])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[94])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[95])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[96])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[97])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[98])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[99])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[100])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[101])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[102])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[103])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[104])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[105])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[106])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[107])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[108])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[109])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[110])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[111])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[112])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[113])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[114])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[115])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[116])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[117])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[118])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[119])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[120])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[121])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[122])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[123])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[124])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[125])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[126])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[127]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x64b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x128b.x1.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b(
  std::uint32_t taddr,
  const B32 (&values)[2]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b(
  std::uint32_t taddr,
  const B32 (&values)[2])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x128b.x1.b32 [%0], {%1, %2};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x128b.x1.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[2]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x128b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[2])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x128b.x1.unpack::16b.b32 [%0], {%1, %2};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x128b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x128b.x2.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b(
  std::uint32_t taddr,
  const B32 (&values)[4]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b(
  std::uint32_t taddr,
  const B32 (&values)[4])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x128b.x2.b32 [%0], {%1, %2, %3, %4};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x128b.x2.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[4]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x128b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[4])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x128b.x2.unpack::16b.b32 [%0], {%1, %2, %3, %4};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x128b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x128b.x4.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b(
  std::uint32_t taddr,
  const B32 (&values)[8]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b(
  std::uint32_t taddr,
  const B32 (&values)[8])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x128b.x4.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x128b.x4.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[8]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x128b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[8])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x128b.x4.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x128b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x128b.x8.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b(
  std::uint32_t taddr,
  const B32 (&values)[16]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b(
  std::uint32_t taddr,
  const B32 (&values)[16])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x128b.x8.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x128b.x8.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[16]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x128b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[16])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x128b.x8.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x128b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x128b.x16.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b(
  std::uint32_t taddr,
  const B32 (&values)[32]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b(
  std::uint32_t taddr,
  const B32 (&values)[32])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x128b.x16.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x128b.x16.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[32]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x128b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[32])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x128b.x16.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x128b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x128b.x32.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b(
  std::uint32_t taddr,
  const B32 (&values)[64]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b(
  std::uint32_t taddr,
  const B32 (&values)[64])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x128b.x32.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x128b.x32.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[64]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x128b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[64])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x128b.x32.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x128b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x128b.x64.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b(
  std::uint32_t taddr,
  const B32 (&values)[128]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b(
  std::uint32_t taddr,
  const B32 (&values)[128])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x128b.x64.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127, %128};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[64])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[65])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[66])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[67])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[68])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[69])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[70])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[71])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[72])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[73])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[74])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[75])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[76])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[77])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[78])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[79])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[80])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[81])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[82])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[83])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[84])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[85])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[86])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[87])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[88])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[89])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[90])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[91])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[92])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[93])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[94])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[95])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[96])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[97])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[98])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[99])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[100])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[101])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[102])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[103])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[104])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[105])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[106])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[107])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[108])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[109])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[110])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[111])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[112])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[113])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[114])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[115])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[116])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[117])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[118])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[119])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[120])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[121])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[122])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[123])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[124])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[125])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[126])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[127]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x128b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x128b.x64.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[128]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x128b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x128b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[128])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x128b.x64.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127, %128};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[64])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[65])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[66])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[67])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[68])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[69])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[70])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[71])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[72])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[73])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[74])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[75])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[76])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[77])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[78])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[79])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[80])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[81])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[82])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[83])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[84])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[85])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[86])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[87])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[88])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[89])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[90])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[91])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[92])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[93])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[94])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[95])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[96])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[97])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[98])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[99])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[100])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[101])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[102])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[103])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[104])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[105])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[106])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[107])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[108])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[109])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[110])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[111])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[112])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[113])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[114])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[115])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[116])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[117])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[118])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[119])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[120])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[121])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[122])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[123])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[124])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[125])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[126])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[127]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x128b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x256b.x1.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b(
  std::uint32_t taddr,
  const B32 (&values)[4]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b(
  std::uint32_t taddr,
  const B32 (&values)[4])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x256b.x1.b32 [%0], {%1, %2, %3, %4};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x256b.x1.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[4]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x256b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[4])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x256b.x1.unpack::16b.b32 [%0], {%1, %2, %3, %4};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x256b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x256b.x2.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b(
  std::uint32_t taddr,
  const B32 (&values)[8]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b(
  std::uint32_t taddr,
  const B32 (&values)[8])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x256b.x2.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x256b.x2.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[8]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x256b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[8])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x256b.x2.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x256b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x256b.x4.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b(
  std::uint32_t taddr,
  const B32 (&values)[16]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b(
  std::uint32_t taddr,
  const B32 (&values)[16])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x256b.x4.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x256b.x4.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[16]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x256b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[16])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x256b.x4.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x256b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x256b.x8.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b(
  std::uint32_t taddr,
  const B32 (&values)[32]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b(
  std::uint32_t taddr,
  const B32 (&values)[32])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x256b.x8.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x256b.x8.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[32]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x256b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[32])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x256b.x8.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x256b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x256b.x16.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b(
  std::uint32_t taddr,
  const B32 (&values)[64]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b(
  std::uint32_t taddr,
  const B32 (&values)[64])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x256b.x16.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x256b.x16.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[64]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x256b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[64])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x256b.x16.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x256b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x256b.x32.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b(
  std::uint32_t taddr,
  const B32 (&values)[128]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b(
  std::uint32_t taddr,
  const B32 (&values)[128])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x256b.x32.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127, %128};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[64])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[65])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[66])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[67])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[68])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[69])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[70])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[71])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[72])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[73])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[74])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[75])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[76])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[77])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[78])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[79])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[80])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[81])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[82])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[83])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[84])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[85])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[86])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[87])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[88])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[89])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[90])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[91])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[92])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[93])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[94])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[95])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[96])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[97])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[98])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[99])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[100])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[101])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[102])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[103])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[104])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[105])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[106])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[107])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[108])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[109])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[110])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[111])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[112])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[113])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[114])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[115])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[116])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[117])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[118])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[119])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[120])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[121])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[122])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[123])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[124])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[125])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[126])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[127]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x256b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x256b.x32.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[128]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x256b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x256b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[128])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x256b.x32.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127, %128};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[64])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[65])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[66])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[67])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[68])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[69])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[70])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[71])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[72])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[73])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[74])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[75])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[76])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[77])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[78])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[79])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[80])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[81])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[82])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[83])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[84])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[85])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[86])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[87])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[88])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[89])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[90])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[91])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[92])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[93])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[94])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[95])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[96])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[97])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[98])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[99])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[100])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[101])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[102])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[103])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[104])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[105])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[106])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[107])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[108])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[109])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[110])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[111])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[112])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[113])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[114])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[115])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[116])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[117])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[118])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[119])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[120])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[121])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[122])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[123])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[124])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[125])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[126])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[127]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x256b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.32x32b.x1.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b(
  std::uint32_t taddr,
  const B32 (&values)[1]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b(
  std::uint32_t taddr,
  const B32 (&values)[1])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.32x32b.x1.b32 [%0], {%1};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.32x32b.x1.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[1]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_32x32b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[1])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.32x32b.x1.unpack::16b.b32 [%0], {%1};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_32x32b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.32x32b.x2.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b(
  std::uint32_t taddr,
  const B32 (&values)[2]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b(
  std::uint32_t taddr,
  const B32 (&values)[2])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.32x32b.x2.b32 [%0], {%1, %2};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.32x32b.x2.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[2]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_32x32b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[2])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.32x32b.x2.unpack::16b.b32 [%0], {%1, %2};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_32x32b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.32x32b.x4.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b(
  std::uint32_t taddr,
  const B32 (&values)[4]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b(
  std::uint32_t taddr,
  const B32 (&values)[4])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.32x32b.x4.b32 [%0], {%1, %2, %3, %4};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.32x32b.x4.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[4]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_32x32b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[4])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.32x32b.x4.unpack::16b.b32 [%0], {%1, %2, %3, %4};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_32x32b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.32x32b.x8.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b(
  std::uint32_t taddr,
  const B32 (&values)[8]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b(
  std::uint32_t taddr,
  const B32 (&values)[8])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.32x32b.x8.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.32x32b.x8.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[8]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_32x32b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[8])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.32x32b.x8.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_32x32b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.32x32b.x16.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b(
  std::uint32_t taddr,
  const B32 (&values)[16]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b(
  std::uint32_t taddr,
  const B32 (&values)[16])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.32x32b.x16.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.32x32b.x16.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[16]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_32x32b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[16])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.32x32b.x16.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_32x32b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.32x32b.x32.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b(
  std::uint32_t taddr,
  const B32 (&values)[32]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b(
  std::uint32_t taddr,
  const B32 (&values)[32])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.32x32b.x32.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.32x32b.x32.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[32]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_32x32b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[32])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.32x32b.x32.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_32x32b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.32x32b.x64.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b(
  std::uint32_t taddr,
  const B32 (&values)[64]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b(
  std::uint32_t taddr,
  const B32 (&values)[64])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.32x32b.x64.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.32x32b.x64.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[64]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_32x32b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[64])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.32x32b.x64.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_32x32b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.32x32b.x128.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b(
  std::uint32_t taddr,
  const B32 (&values)[128]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b(
  std::uint32_t taddr,
  const B32 (&values)[128])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.32x32b.x128.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127, %128};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[64])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[65])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[66])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[67])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[68])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[69])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[70])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[71])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[72])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[73])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[74])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[75])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[76])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[77])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[78])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[79])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[80])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[81])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[82])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[83])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[84])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[85])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[86])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[87])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[88])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[89])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[90])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[91])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[92])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[93])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[94])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[95])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[96])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[97])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[98])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[99])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[100])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[101])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[102])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[103])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[104])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[105])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[106])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[107])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[108])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[109])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[110])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[111])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[112])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[113])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[114])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[115])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[116])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[117])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[118])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[119])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[120])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[121])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[122])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[123])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[124])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[125])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[126])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[127]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_32x32b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.32x32b.x128.unpack::16b.b32 [taddr], values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[128]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_32x32b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_32x32b_unpack_16b(
  std::uint32_t taddr,
  const B32 (&values)[128])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.32x32b.x128.unpack::16b.b32 [%0], {%1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127, %128};"
      :
      : "r"(taddr),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[64])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[65])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[66])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[67])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[68])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[69])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[70])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[71])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[72])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[73])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[74])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[75])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[76])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[77])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[78])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[79])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[80])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[81])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[82])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[83])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[84])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[85])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[86])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[87])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[88])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[89])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[90])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[91])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[92])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[93])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[94])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[95])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[96])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[97])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[98])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[99])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[100])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[101])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[102])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[103])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[104])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[105])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[106])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[107])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[108])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[109])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[110])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[111])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[112])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[113])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[114])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[115])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[116])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[117])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[118])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[119])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[120])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[121])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[122])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[123])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[124])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[125])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[126])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[127]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_32x32b_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x32bx2.x1.b32 [taddr], immHalfSplitoff, values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[1]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[1])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x32bx2.x1.b32 [%0], %1, {%2};"
      :
      : "r"(taddr),
        "n"(immHalfSplitoff.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x32bx2.x1.unpack::16b.b32 [taddr], immHalfSplitoff, values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2_unpack_16b(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[1]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x32bx2_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2_unpack_16b(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[1])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x32bx2.x1.unpack::16b.b32 [%0], %1, {%2};"
      :
      : "r"(taddr),
        "n"(immHalfSplitoff.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x32bx2_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x32bx2.x2.b32 [taddr], immHalfSplitoff, values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[2]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[2])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x32bx2.x2.b32 [%0], %1, {%2, %3};"
      :
      : "r"(taddr),
        "n"(immHalfSplitoff.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x32bx2.x2.unpack::16b.b32 [taddr], immHalfSplitoff, values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2_unpack_16b(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[2]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x32bx2_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2_unpack_16b(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[2])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x32bx2.x2.unpack::16b.b32 [%0], %1, {%2, %3};"
      :
      : "r"(taddr),
        "n"(immHalfSplitoff.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x32bx2_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x32bx2.x4.b32 [taddr], immHalfSplitoff, values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[4]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[4])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x32bx2.x4.b32 [%0], %1, {%2, %3, %4, %5};"
      :
      : "r"(taddr),
        "n"(immHalfSplitoff.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x32bx2.x4.unpack::16b.b32 [taddr], immHalfSplitoff, values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2_unpack_16b(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[4]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x32bx2_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2_unpack_16b(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[4])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x32bx2.x4.unpack::16b.b32 [%0], %1, {%2, %3, %4, %5};"
      :
      : "r"(taddr),
        "n"(immHalfSplitoff.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x32bx2_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x32bx2.x8.b32 [taddr], immHalfSplitoff, values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[8]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[8])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x32bx2.x8.b32 [%0], %1, {%2, %3, %4, %5, %6, %7, %8, %9};"
      :
      : "r"(taddr),
        "n"(immHalfSplitoff.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x32bx2.x8.unpack::16b.b32 [taddr], immHalfSplitoff, values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2_unpack_16b(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[8]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x32bx2_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2_unpack_16b(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[8])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x32bx2.x8.unpack::16b.b32 [%0], %1, {%2, %3, %4, %5, %6, %7, %8, %9};"
      :
      : "r"(taddr),
        "n"(immHalfSplitoff.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x32bx2_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x32bx2.x16.b32 [taddr], immHalfSplitoff, values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[16]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[16])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x32bx2.x16.b32 [%0], %1, {%2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17};"
      :
      : "r"(taddr),
        "n"(immHalfSplitoff.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x32bx2.x16.unpack::16b.b32 [taddr], immHalfSplitoff, values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2_unpack_16b(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[16]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x32bx2_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2_unpack_16b(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[16])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x32bx2.x16.unpack::16b.b32 [%0], %1, {%2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17};"
      :
      : "r"(taddr),
        "n"(immHalfSplitoff.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x32bx2_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x32bx2.x32.b32 [taddr], immHalfSplitoff, values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[32]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[32])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x32bx2.x32.b32 [%0], %1, {%2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33};"
      :
      : "r"(taddr),
        "n"(immHalfSplitoff.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x32bx2.x32.unpack::16b.b32 [taddr], immHalfSplitoff, values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2_unpack_16b(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[32]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x32bx2_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2_unpack_16b(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[32])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x32bx2.x32.unpack::16b.b32 [%0], %1, {%2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33};"
      :
      : "r"(taddr),
        "n"(immHalfSplitoff.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x32bx2_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x32bx2.x64.b32 [taddr], immHalfSplitoff, values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[64]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[64])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x32bx2.x64.b32 [%0], %1, {%2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65};"
      :
      : "r"(taddr),
        "n"(immHalfSplitoff.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x32bx2.x64.unpack::16b.b32 [taddr], immHalfSplitoff, values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2_unpack_16b(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[64]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x32bx2_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2_unpack_16b(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[64])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x32bx2.x64.unpack::16b.b32 [%0], %1, {%2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65};"
      :
      : "r"(taddr),
        "n"(immHalfSplitoff.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x32bx2_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x32bx2.x128.b32 [taddr], immHalfSplitoff, values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[128]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[128])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x32bx2.x128.b32 [%0], %1, {%2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127, %128, %129};"
      :
      : "r"(taddr),
        "n"(immHalfSplitoff.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[64])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[65])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[66])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[67])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[68])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[69])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[70])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[71])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[72])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[73])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[74])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[75])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[76])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[77])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[78])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[79])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[80])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[81])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[82])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[83])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[84])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[85])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[86])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[87])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[88])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[89])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[90])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[91])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[92])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[93])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[94])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[95])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[96])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[97])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[98])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[99])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[100])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[101])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[102])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[103])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[104])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[105])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[106])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[107])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[108])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[109])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[110])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[111])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[112])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[113])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[114])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[115])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[116])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[117])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[118])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[119])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[120])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[121])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[122])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[123])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[124])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[125])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[126])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[127]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x32bx2_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.st.sync.aligned.16x32bx2.x128.unpack::16b.b32 [taddr], immHalfSplitoff, values; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2_unpack_16b(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[128]);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_st_16x32bx2_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tcgen05_st_16x32bx2_unpack_16b(
  std::uint32_t taddr,
  cuda_ptx::n32_t<N32> immHalfSplitoff,
  const B32 (&values)[128])
{
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm (
      "tcgen05.st.sync.aligned.16x32bx2.x128.unpack::16b.b32 [%0], %1, {%2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31, %32, %33, %34, %35, %36, %37, %38, %39, %40, %41, %42, %43, %44, %45, %46, %47, %48, %49, %50, %51, %52, %53, %54, %55, %56, %57, %58, %59, %60, %61, %62, %63, %64, %65, %66, %67, %68, %69, %70, %71, %72, %73, %74, %75, %76, %77, %78, %79, %80, %81, %82, %83, %84, %85, %86, %87, %88, %89, %90, %91, %92, %93, %94, %95, %96, %97, %98, %99, %100, %101, %102, %103, %104, %105, %106, %107, %108, %109, %110, %111, %112, %113, %114, %115, %116, %117, %118, %119, %120, %121, %122, %123, %124, %125, %126, %127, %128, %129};"
      :
      : "r"(taddr),
        "n"(immHalfSplitoff.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[0])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[1])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[2])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[3])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[4])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[5])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[6])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[7])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[8])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[9])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[10])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[11])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[12])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[13])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[14])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[15])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[16])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[17])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[18])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[19])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[20])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[21])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[22])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[23])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[24])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[25])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[26])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[27])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[28])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[29])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[30])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[31])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[32])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[33])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[34])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[35])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[36])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[37])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[38])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[39])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[40])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[41])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[42])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[43])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[44])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[45])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[46])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[47])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[48])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[49])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[50])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[51])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[52])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[53])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[54])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[55])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[56])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[57])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[58])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[59])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[60])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[61])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[62])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[63])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[64])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[65])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[66])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[67])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[68])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[69])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[70])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[71])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[72])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[73])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[74])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[75])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[76])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[77])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[78])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[79])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[80])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[81])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[82])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[83])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[84])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[85])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[86])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[87])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[88])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[89])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[90])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[91])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[92])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[93])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[94])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[95])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[96])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[97])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[98])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[99])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[100])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[101])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[102])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[103])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[104])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[105])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[106])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[107])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[108])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[109])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[110])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[111])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[112])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[113])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[114])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[115])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[116])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[117])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[118])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[119])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[120])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[121])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[122])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[123])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[124])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[125])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[126])),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&values[127]))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_st_16x32bx2_unpack_16b_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

} // namespace cuda_ptx

#endif // INSTRUCTIONS_TCGEN05_ST_H

// End of ./instructions/tcgen05_st.h
// Start of ./instructions/tcgen05_wait.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_TCGEN05_WAIT_H
#define INSTRUCTIONS_TCGEN05_WAIT_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/tcgen05_wait.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// tcgen05.wait::ld.sync.aligned; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename = void>
__device__ static inline void tcgen05_wait_ld();
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_wait_ld_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename = void>
__device__ static inline void tcgen05_wait_ld()
{
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm volatile (
      "tcgen05.wait::ld.sync.aligned;"
      :
      :
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_wait_ld_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tcgen05.wait::st.sync.aligned; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f
template <typename = void>
__device__ static inline void tcgen05_wait_st();
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tcgen05_wait_st_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
template <typename = void>
__device__ static inline void tcgen05_wait_st()
{
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110)
    asm volatile (
      "tcgen05.wait::st.sync.aligned;"
      :
      :
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tcgen05_wait_st_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

} // namespace cuda_ptx

#endif // INSTRUCTIONS_TCGEN05_WAIT_H

// End of ./instructions/tcgen05_wait.h
// Start of ./instructions/tensormap_cp_fenceproxy.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_TENSORMAP_CP_FENCEPROXY_H
#define INSTRUCTIONS_TENSORMAP_CP_FENCEPROXY_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/tensormap_cp_fenceproxy.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.sem.scope.sync.aligned  [dst], [src], size; // PTX ISA 83, SM_90
// .sem       = { .release }
// .scope     = { .cta, .cluster, .gpu, .sys }
template <int N32, cuda_ptx::dot_scope Scope>
__device__ static inline void tensormap_cp_fenceproxy(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_t<Scope> scope,
  void* dst,
  const void* src,
  cuda_ptx::n32_t<N32> size);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_cp_fenceproxy_is_not_supported_before_SM_90__();
template <int N32, cuda_ptx::dot_scope Scope>
__device__ static inline void tensormap_cp_fenceproxy(
  cuda_ptx::sem_release_t,
  cuda_ptx::scope_t<Scope> scope,
  void* dst,
  const void* src,
  cuda_ptx::n32_t<N32> size)
{
  // sem == sem_release (due to parameter type constraint)
  static_assert(scope == scope_cta || scope == scope_cluster || scope == scope_gpu || scope == scope_sys, "");
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 900
    if constexpr (scope == scope_cta) {
      asm volatile (
        "tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.cta.sync.aligned  [%0], [%1], %2;"
        :
        : "l"(__as_ptr_gmem(dst)),
          "r"(__as_ptr_smem(src)),
          "n"(size.value)
        : "memory"
      );
    } else if constexpr (scope == scope_cluster) {
      asm volatile (
        "tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.cluster.sync.aligned  [%0], [%1], %2;"
        :
        : "l"(__as_ptr_gmem(dst)),
          "r"(__as_ptr_smem(src)),
          "n"(size.value)
        : "memory"
      );
    } else if constexpr (scope == scope_gpu) {
      asm volatile (
        "tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.gpu.sync.aligned  [%0], [%1], %2;"
        :
        : "l"(__as_ptr_gmem(dst)),
          "r"(__as_ptr_smem(src)),
          "n"(size.value)
        : "memory"
      );
    } else if constexpr (scope == scope_sys) {
      asm volatile (
        "tensormap.cp_fenceproxy.global.shared::cta.tensormap::generic.release.sys.sync.aligned  [%0], [%1], %2;"
        :
        : "l"(__as_ptr_gmem(dst)),
          "r"(__as_ptr_smem(src)),
          "n"(size.value)
        : "memory"
      );
    }
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_cp_fenceproxy_is_not_supported_before_SM_90__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

} // namespace cuda_ptx

#endif // INSTRUCTIONS_TENSORMAP_CP_FENCEPROXY_H

// End of ./instructions/tensormap_cp_fenceproxy.h
// Start of ./instructions/tensormap_replace.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_TENSORMAP_REPLACE_H
#define INSTRUCTIONS_TENSORMAP_REPLACE_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/tensormap_replace.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// tensormap.replace.tile.global_address.space.b1024.b64 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .global }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void tensormap_replace_global_address(
  cuda_ptx::space_global_t,
  void* tm_addr,
  B64 new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_global_address_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void tensormap_replace_global_address(
  cuda_ptx::space_global_t,
  void* tm_addr,
  B64 new_val)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.global_address.global.b1024.b64 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(tm_addr)),
        "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&new_val))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_global_address_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.global_address.space.b1024.b64 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .shared::cta }
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void tensormap_replace_global_address(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  B64 new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_global_address_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void tensormap_replace_global_address(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  B64 new_val)
{
  // space == space_shared (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.global_address.shared::cta.b1024.b64 [%0], %1;"
      :
      : "r"(__as_ptr_smem(tm_addr)),
        "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&new_val))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_global_address_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.rank.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .global }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_rank(
  cuda_ptx::space_global_t,
  void* tm_addr,
  B32 new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_rank_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_rank(
  cuda_ptx::space_global_t,
  void* tm_addr,
  B32 new_val)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.rank.global.b1024.b32 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(tm_addr)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&new_val))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_rank_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.rank.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .shared::cta }
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_rank(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  B32 new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_rank_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_rank(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  B32 new_val)
{
  // space == space_shared (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.rank.shared::cta.b1024.b32 [%0], %1;"
      :
      : "r"(__as_ptr_smem(tm_addr)),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&new_val))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_rank_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.box_dim.space.b1024.b32 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .global }
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_box_dim(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B32 new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_box_dim_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_box_dim(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B32 new_val)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.box_dim.global.b1024.b32 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(tm_addr)),
        "n"(ord.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&new_val))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_box_dim_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.box_dim.space.b1024.b32 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .shared::cta }
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_box_dim(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B32 new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_box_dim_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_box_dim(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B32 new_val)
{
  // space == space_shared (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.box_dim.shared::cta.b1024.b32 [%0], %1, %2;"
      :
      : "r"(__as_ptr_smem(tm_addr)),
        "n"(ord.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&new_val))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_box_dim_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.global_dim.space.b1024.b32 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .global }
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_global_dim(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B32 new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_global_dim_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_global_dim(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B32 new_val)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.global_dim.global.b1024.b32 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(tm_addr)),
        "n"(ord.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&new_val))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_global_dim_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.global_dim.space.b1024.b32 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .shared::cta }
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_global_dim(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B32 new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_global_dim_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_global_dim(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B32 new_val)
{
  // space == space_shared (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.global_dim.shared::cta.b1024.b32 [%0], %1, %2;"
      :
      : "r"(__as_ptr_smem(tm_addr)),
        "n"(ord.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&new_val))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_global_dim_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.global_stride.space.b1024.b64 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .global }
template <int N32, typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void tensormap_replace_global_stride(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B64 new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_global_stride_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32, typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void tensormap_replace_global_stride(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B64 new_val)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.global_stride.global.b1024.b64 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(tm_addr)),
        "n"(ord.value),
        "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&new_val))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_global_stride_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.global_stride.space.b1024.b64 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .shared::cta }
template <int N32, typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void tensormap_replace_global_stride(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B64 new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_global_stride_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32, typename B64, std::enable_if_t<sizeof(B64) == 8, bool> = true>
__device__ static inline void tensormap_replace_global_stride(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B64 new_val)
{
  // space == space_shared (due to parameter type constraint)
  static_assert(sizeof(B64) == 8, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.global_stride.shared::cta.b1024.b64 [%0], %1, %2;"
      :
      : "r"(__as_ptr_smem(tm_addr)),
        "n"(ord.value),
        "l"(/*as_b64*/ *reinterpret_cast<const std::int64_t*>(&new_val))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_global_stride_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.element_stride.space.b1024.b32 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .global }
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_element_stride(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B32 new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_element_stride_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_element_stride(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B32 new_val)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.element_stride.global.b1024.b32 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(tm_addr)),
        "n"(ord.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&new_val))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_element_stride_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.element_stride.space.b1024.b32 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .shared::cta }
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_element_stride(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B32 new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_element_stride_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_element_stride(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B32 new_val)
{
  // space == space_shared (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.element_stride.shared::cta.b1024.b32 [%0], %1, %2;"
      :
      : "r"(__as_ptr_smem(tm_addr)),
        "n"(ord.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&new_val))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_element_stride_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.element_stride.space.b1024.b32 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .global }
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_element_size(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B32 new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_element_size_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_element_size(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B32 new_val)
{
  // space == space_global (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.element_stride.global.b1024.b32 [%0], %1, %2;"
      :
      : "l"(__as_ptr_gmem(tm_addr)),
        "n"(ord.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&new_val))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_element_size_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.element_stride.space.b1024.b32 [tm_addr], ord, new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .shared::cta }
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_element_size(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B32 new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_element_size_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32, typename B32, std::enable_if_t<sizeof(B32) == 4, bool> = true>
__device__ static inline void tensormap_replace_element_size(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> ord,
  B32 new_val)
{
  // space == space_shared (due to parameter type constraint)
  static_assert(sizeof(B32) == 4, "");
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.element_stride.shared::cta.b1024.b32 [%0], %1, %2;"
      :
      : "r"(__as_ptr_smem(tm_addr)),
        "n"(ord.value),
        "r"(/*as_b32*/ *reinterpret_cast<const std::int32_t*>(&new_val))
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_element_size_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.elemtype.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .global }
template <int N32>
__device__ static inline void tensormap_replace_elemtype(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_elemtype_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32>
__device__ static inline void tensormap_replace_elemtype(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val)
{
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.elemtype.global.b1024.b32 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(tm_addr)),
        "n"(new_val.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_elemtype_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.elemtype.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .shared::cta }
template <int N32>
__device__ static inline void tensormap_replace_elemtype(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_elemtype_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32>
__device__ static inline void tensormap_replace_elemtype(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val)
{
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.elemtype.shared::cta.b1024.b32 [%0], %1;"
      :
      : "r"(__as_ptr_smem(tm_addr)),
        "n"(new_val.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_elemtype_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.interleave_layout.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .global }
template <int N32>
__device__ static inline void tensormap_replace_interleave_layout(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_interleave_layout_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32>
__device__ static inline void tensormap_replace_interleave_layout(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val)
{
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.interleave_layout.global.b1024.b32 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(tm_addr)),
        "n"(new_val.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_interleave_layout_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.interleave_layout.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .shared::cta }
template <int N32>
__device__ static inline void tensormap_replace_interleave_layout(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_interleave_layout_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32>
__device__ static inline void tensormap_replace_interleave_layout(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val)
{
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.interleave_layout.shared::cta.b1024.b32 [%0], %1;"
      :
      : "r"(__as_ptr_smem(tm_addr)),
        "n"(new_val.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_interleave_layout_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.swizzle_mode.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .global }
template <int N32>
__device__ static inline void tensormap_replace_swizzle_mode(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_swizzle_mode_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32>
__device__ static inline void tensormap_replace_swizzle_mode(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val)
{
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.swizzle_mode.global.b1024.b32 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(tm_addr)),
        "n"(new_val.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_swizzle_mode_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.swizzle_mode.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .shared::cta }
template <int N32>
__device__ static inline void tensormap_replace_swizzle_mode(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_swizzle_mode_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32>
__device__ static inline void tensormap_replace_swizzle_mode(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val)
{
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.swizzle_mode.shared::cta.b1024.b32 [%0], %1;"
      :
      : "r"(__as_ptr_smem(tm_addr)),
        "n"(new_val.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_swizzle_mode_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.fill_mode.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .global }
template <int N32>
__device__ static inline void tensormap_replace_fill_mode(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_fill_mode_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32>
__device__ static inline void tensormap_replace_fill_mode(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val)
{
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.fill_mode.global.b1024.b32 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(tm_addr)),
        "n"(new_val.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_fill_mode_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.fill_mode.space.b1024.b32 [tm_addr], new_val; // PTX ISA 83, SM_90a, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .shared::cta }
template <int N32>
__device__ static inline void tensormap_replace_fill_mode(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val);
*/
#if __libcuda_ptx_isa >= 830
extern "C" __device__ void __cuda_ptx_tensormap_replace_fill_mode_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32>
__device__ static inline void tensormap_replace_fill_mode(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val)
{
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 900) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.fill_mode.shared::cta.b1024.b32 [%0], %1;"
      :
      : "r"(__as_ptr_smem(tm_addr)),
        "n"(new_val.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_fill_mode_is_only_supported_on_SM_90a_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 830

/*
// tensormap.replace.tile.swizzle_atomicity.space.b1024.b32 [tm_addr], new_val; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .global }
template <int N32>
__device__ static inline void tensormap_replace_swizzle_atomicity(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tensormap_replace_swizzle_atomicity_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32>
__device__ static inline void tensormap_replace_swizzle_atomicity(
  cuda_ptx::space_global_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val)
{
  // space == space_global (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.swizzle_atomicity.global.b1024.b32 [%0], %1;"
      :
      : "l"(__as_ptr_gmem(tm_addr)),
        "n"(new_val.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_swizzle_atomicity_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

/*
// tensormap.replace.tile.swizzle_atomicity.space.b1024.b32 [tm_addr], new_val; // PTX ISA 86, SM_100a, SM_100f, SM_103a, SM_103f, SM_110a, SM_110f, SM_120a, SM_120f, SM_121a, SM_121f
// .space     = { .shared::cta }
template <int N32>
__device__ static inline void tensormap_replace_swizzle_atomicity(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val);
*/
#if __libcuda_ptx_isa >= 860
extern "C" __device__ void __cuda_ptx_tensormap_replace_swizzle_atomicity_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
template <int N32>
__device__ static inline void tensormap_replace_swizzle_atomicity(
  cuda_ptx::space_shared_t,
  void* tm_addr,
  cuda_ptx::n32_t<N32> new_val)
{
  // space == space_shared (due to parameter type constraint)
  #if defined(_NVHPC_CUDA) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1000) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1030) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1100) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1200) || (_LIBCUDA_PTX_ARCH_SPECIFIC() == 1210) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(100) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(103) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(110) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(120) || __CUDA_HAS_ARCH_FAMILY_SPECIFIC(121)
    asm (
      "tensormap.replace.tile.swizzle_atomicity.shared::cta.b1024.b32 [%0], %1;"
      :
      : "r"(__as_ptr_smem(tm_addr)),
        "n"(new_val.value)
      : "memory"
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_tensormap_replace_swizzle_atomicity_is_only_supported_on_SM_100a_100f_103a_103f_110a_110f_120a_120f_121a_121f__();
  #endif
}
#endif // __libcuda_ptx_isa >= 860

} // namespace cuda_ptx

#endif // INSTRUCTIONS_TENSORMAP_REPLACE_H

// End of ./instructions/tensormap_replace.h
// Start of ./instructions/trap.h
// -*- cuda -*-

#ifndef INSTRUCTIONS_TRAP_H
#define INSTRUCTIONS_TRAP_H

// We don't support including this header directly. It must be included through
// ptx.h.
#ifndef LIBCUDA_PTX_H
#error "instructions/trap.h should only be included in from ptx.h"
#endif // LIBCUDA_PTX_H

namespace cuda_ptx {

/*
// trap; // PTX ISA 10, SM_50
template <typename = void>
__device__ static inline void trap();
*/
#if __libcuda_ptx_isa >= 100
extern "C" __device__ void __cuda_ptx_trap_is_not_supported_before_SM_50__();
template <typename = void>
__device__ static inline void trap()
{
  #if defined(_NVHPC_CUDA) || __CUDA_ARCH__ >= 500
    asm volatile (
      "trap;"
      :
      :
      :
    );
  #else
    // Unsupported architectures will have a linker error with a semi-decent error message
    __cuda_ptx_trap_is_not_supported_before_SM_50__();
  #endif
}
#endif // __libcuda_ptx_isa >= 100

} // namespace cuda_ptx

#endif // INSTRUCTIONS_TRAP_H

// End of ./instructions/trap.h

#endif // LIBCUDA_PTX_H