45 #ifndef KOKKOS_HIPSPACE_HPP 46 #define KOKKOS_HIPSPACE_HPP 48 #include <Kokkos_Core_fwd.hpp> 50 #if defined(KOKKOS_ENABLE_HIP) 58 #include <Kokkos_HostSpace.hpp> 60 #include <Kokkos_ScratchSpace.hpp> 62 #include <impl/Kokkos_Profiling_Interface.hpp> 63 #include <impl/Kokkos_ExecSpaceInitializer.hpp> 64 #include <impl/Kokkos_HostSharedPtr.hpp> 66 #include <hip/hip_runtime_api.h> 76 using memory_space = HIPSpace;
77 using execution_space = Kokkos::Experimental::HIP;
78 using device_type = Kokkos::Device<execution_space, memory_space>;
80 using size_type =
unsigned int;
85 HIPSpace(HIPSpace&& rhs) =
default;
86 HIPSpace(
const HIPSpace& rhs) =
default;
87 HIPSpace& operator=(HIPSpace&& rhs) =
default;
88 HIPSpace& operator=(
const HIPSpace& rhs) =
default;
89 ~HIPSpace() =
default;
92 void* allocate(
const size_t arg_alloc_size)
const;
93 void* allocate(
const char* arg_label,
const size_t arg_alloc_size,
94 const size_t arg_logical_size = 0)
const;
97 void deallocate(
void*
const arg_alloc_ptr,
const size_t arg_alloc_size)
const;
98 void deallocate(
const char* arg_label,
void*
const arg_alloc_ptr,
99 const size_t arg_alloc_size,
100 const size_t arg_logical_size = 0)
const;
103 template <
class,
class,
class,
class>
104 friend class LogicalMemorySpace;
105 void* impl_allocate(
const char* arg_label,
const size_t arg_alloc_size,
106 const size_t arg_logical_size = 0,
107 const Kokkos::Tools::SpaceHandle =
108 Kokkos::Tools::make_space_handle(name()))
const;
109 void impl_deallocate(
const char* arg_label,
void*
const arg_alloc_ptr,
110 const size_t arg_alloc_size,
111 const size_t arg_logical_size = 0,
112 const Kokkos::Tools::SpaceHandle =
113 Kokkos::Tools::make_space_handle(name()))
const;
117 static constexpr
const char* name() {
return "HIP"; }
121 KOKKOS_DEPRECATED
static void access_error();
122 KOKKOS_DEPRECATED
static void access_error(
const void*
const);
127 friend class Kokkos::Impl::SharedAllocationRecord<
128 Kokkos::Experimental::HIPSpace, void>;
142 class HIPHostPinnedSpace {
146 using execution_space = HostSpace::execution_space;
147 using memory_space = HIPHostPinnedSpace;
148 using device_type = Kokkos::Device<execution_space, memory_space>;
149 using size_type =
unsigned int;
153 HIPHostPinnedSpace();
154 HIPHostPinnedSpace(HIPHostPinnedSpace&& rhs) =
default;
155 HIPHostPinnedSpace(
const HIPHostPinnedSpace& rhs) =
default;
156 HIPHostPinnedSpace& operator=(HIPHostPinnedSpace&& rhs) =
default;
157 HIPHostPinnedSpace& operator=(
const HIPHostPinnedSpace& rhs) =
default;
158 ~HIPHostPinnedSpace() =
default;
161 void* allocate(
const size_t arg_alloc_size)
const;
162 void* allocate(
const char* arg_label,
const size_t arg_alloc_size,
163 const size_t arg_logical_size = 0)
const;
166 void deallocate(
void*
const arg_alloc_ptr,
const size_t arg_alloc_size)
const;
167 void deallocate(
const char* arg_label,
void*
const arg_alloc_ptr,
168 const size_t arg_alloc_size,
169 const size_t arg_logical_size = 0)
const;
172 template <
class,
class,
class,
class>
173 friend class LogicalMemorySpace;
174 void* impl_allocate(
const char* arg_label,
const size_t arg_alloc_size,
175 const size_t arg_logical_size = 0,
176 const Kokkos::Tools::SpaceHandle =
177 Kokkos::Tools::make_space_handle(name()))
const;
178 void impl_deallocate(
const char* arg_label,
void*
const arg_alloc_ptr,
179 const size_t arg_alloc_size,
180 const size_t arg_logical_size = 0,
181 const Kokkos::Tools::SpaceHandle =
182 Kokkos::Tools::make_space_handle(name()))
const;
186 static constexpr
const char* name() {
return "HIPHostPinned"; }
201 Kokkos::Experimental::HIPSpace>::assignable,
207 struct MemorySpaceAccess<
Kokkos::HostSpace, Kokkos::Experimental::HIPSpace> {
208 enum :
bool { assignable =
false };
209 enum :
bool { accessible =
false };
210 enum :
bool { deepcopy =
true };
214 struct MemorySpaceAccess<
Kokkos::HostSpace,
215 Kokkos::Experimental::HIPHostPinnedSpace> {
217 enum :
bool { assignable =
true };
218 enum :
bool { accessible =
true };
219 enum :
bool { deepcopy =
true };
226 enum :
bool { assignable =
false };
227 enum :
bool { accessible =
false };
228 enum :
bool { deepcopy =
true };
232 struct MemorySpaceAccess<
Kokkos::Experimental::HIPSpace,
233 Kokkos::Experimental::HIPHostPinnedSpace> {
235 enum :
bool { assignable =
false };
236 enum :
bool { accessible =
true };
237 enum :
bool { deepcopy =
true };
245 struct MemorySpaceAccess<
Kokkos::Experimental::HIPHostPinnedSpace,
247 enum :
bool { assignable =
false };
248 enum :
bool { accessible =
true };
249 enum :
bool { deepcopy =
true };
253 struct MemorySpaceAccess<
Kokkos::Experimental::HIPHostPinnedSpace,
254 Kokkos::Experimental::HIPSpace> {
255 enum :
bool { assignable =
false };
256 enum :
bool { accessible =
false };
257 enum :
bool { deepcopy =
true };
271 void DeepCopyAsyncHIP(
void* dst,
const void* src,
size_t n);
274 struct DeepCopy<
Kokkos::Experimental::HIPSpace, Kokkos::Experimental::HIPSpace,
275 Kokkos::Experimental::HIP> {
276 DeepCopy(
void* dst,
const void* src,
size_t);
277 DeepCopy(
const Kokkos::Experimental::HIP&,
void* dst,
const void* src,
282 struct DeepCopy<
Kokkos::Experimental::HIPSpace, HostSpace,
283 Kokkos::Experimental::HIP> {
284 DeepCopy(
void* dst,
const void* src,
size_t);
285 DeepCopy(
const Kokkos::Experimental::HIP&,
void* dst,
const void* src,
290 struct DeepCopy<HostSpace,
Kokkos::Experimental::HIPSpace,
291 Kokkos::Experimental::HIP> {
292 DeepCopy(
void* dst,
const void* src,
size_t);
293 DeepCopy(
const Kokkos::Experimental::HIP&,
void* dst,
const void* src,
297 template <
class ExecutionSpace>
298 struct DeepCopy<
Kokkos::Experimental::HIPSpace, Kokkos::Experimental::HIPSpace,
300 inline DeepCopy(
void* dst,
const void* src,
size_t n) {
301 (void)DeepCopy<Kokkos::Experimental::HIPSpace,
302 Kokkos::Experimental::HIPSpace, Kokkos::Experimental::HIP>(
306 inline DeepCopy(
const ExecutionSpace& exec,
void* dst,
const void* src,
309 DeepCopyAsyncHIP(dst, src, n);
313 template <
class ExecutionSpace>
314 struct DeepCopy<
Kokkos::Experimental::HIPSpace, HostSpace, ExecutionSpace> {
315 inline DeepCopy(
void* dst,
const void* src,
size_t n) {
316 (void)DeepCopy<Kokkos::Experimental::HIPSpace, HostSpace,
317 Kokkos::Experimental::HIP>(dst, src, n);
320 inline DeepCopy(
const ExecutionSpace& exec,
void* dst,
const void* src,
323 DeepCopyAsyncHIP(dst, src, n);
327 template <
class ExecutionSpace>
328 struct DeepCopy<HostSpace,
Kokkos::Experimental::HIPSpace, ExecutionSpace> {
329 inline DeepCopy(
void* dst,
const void* src,
size_t n) {
330 (void)DeepCopy<HostSpace, Kokkos::Experimental::HIPSpace,
331 Kokkos::Experimental::HIP>(dst, src, n);
334 inline DeepCopy(
const ExecutionSpace& exec,
void* dst,
const void* src,
337 DeepCopyAsyncHIP(dst, src, n);
342 struct DeepCopy<
Kokkos::Experimental::HIPHostPinnedSpace,
343 Kokkos::Experimental::HIPHostPinnedSpace,
344 Kokkos::Experimental::HIP> {
345 DeepCopy(
void* dst,
const void* src,
size_t);
346 DeepCopy(
const Kokkos::Experimental::HIP&,
void* dst,
const void* src,
351 struct DeepCopy<
Kokkos::Experimental::HIPHostPinnedSpace, HostSpace,
352 Kokkos::Experimental::HIP> {
353 DeepCopy(
void* dst,
const void* src,
size_t);
354 DeepCopy(
const Kokkos::Experimental::HIP&,
void* dst,
const void* src,
359 struct DeepCopy<HostSpace,
Kokkos::Experimental::HIPHostPinnedSpace,
360 Kokkos::Experimental::HIP> {
361 DeepCopy(
void* dst,
const void* src,
size_t);
362 DeepCopy(
const Kokkos::Experimental::HIP&,
void* dst,
const void* src,
366 template <
class ExecutionSpace>
367 struct DeepCopy<
Kokkos::Experimental::HIPSpace,
368 Kokkos::Experimental::HIPHostPinnedSpace, ExecutionSpace> {
369 inline DeepCopy(
void* dst,
const void* src,
size_t n) {
370 (void)DeepCopy<Kokkos::Experimental::HIPSpace, HostSpace,
371 Kokkos::Experimental::HIP>(dst, src, n);
374 inline DeepCopy(
const ExecutionSpace& exec,
void* dst,
const void* src,
377 DeepCopyAsyncHIP(dst, src, n);
381 template <
class ExecutionSpace>
382 struct DeepCopy<
Kokkos::Experimental::HIPHostPinnedSpace,
383 Kokkos::Experimental::HIPSpace, ExecutionSpace> {
384 inline DeepCopy(
void* dst,
const void* src,
size_t n) {
385 (void)DeepCopy<HostSpace, Kokkos::Experimental::HIPSpace,
386 Kokkos::Experimental::HIP>(dst, src, n);
389 inline DeepCopy(
const ExecutionSpace& exec,
void* dst,
const void* src,
392 DeepCopyAsyncHIP(dst, src, n);
396 template <
class ExecutionSpace>
397 struct DeepCopy<
Kokkos::Experimental::HIPHostPinnedSpace,
398 Kokkos::Experimental::HIPHostPinnedSpace, ExecutionSpace> {
399 inline DeepCopy(
void* dst,
const void* src,
size_t n) {
400 (void)DeepCopy<Kokkos::Experimental::HIPHostPinnedSpace,
401 Kokkos::Experimental::HIPHostPinnedSpace,
402 Kokkos::Experimental::HIP>(dst, src, n);
405 inline DeepCopy(
const ExecutionSpace& exec,
void* dst,
const void* src,
408 DeepCopyAsyncHIP(dst, src, n);
412 template <
class ExecutionSpace>
413 struct DeepCopy<
Kokkos::Experimental::HIPHostPinnedSpace, HostSpace,
415 inline DeepCopy(
void* dst,
const void* src,
size_t n) {
416 (void)DeepCopy<Kokkos::Experimental::HIPHostPinnedSpace, HostSpace,
417 Kokkos::Experimental::HIP>(dst, src, n);
420 inline DeepCopy(
const ExecutionSpace& exec,
void* dst,
const void* src,
423 DeepCopyAsyncHIP(dst, src, n);
427 template <
class ExecutionSpace>
428 struct DeepCopy<HostSpace,
Kokkos::Experimental::HIPHostPinnedSpace,
430 inline DeepCopy(
void* dst,
const void* src,
size_t n) {
431 (void)DeepCopy<HostSpace, Kokkos::Experimental::HIPHostPinnedSpace,
432 Kokkos::Experimental::HIP>(dst, src, n);
435 inline DeepCopy(
const ExecutionSpace& exec,
void* dst,
const void* src,
438 DeepCopyAsyncHIP(dst, src, n);
451 class SharedAllocationRecord<
Kokkos::Experimental::HIPSpace, void>
452 :
public HostInaccessibleSharedAllocationRecordCommon<
453 Kokkos::Experimental::HIPSpace> {
455 friend class SharedAllocationRecordCommon<
Kokkos::Experimental::HIPSpace>;
456 friend class HostInaccessibleSharedAllocationRecordCommon<
457 Kokkos::Experimental::HIPSpace>;
458 using base_t = HostInaccessibleSharedAllocationRecordCommon<
459 Kokkos::Experimental::HIPSpace>;
460 using RecordBase = SharedAllocationRecord<void, void>;
462 SharedAllocationRecord(const SharedAllocationRecord&) = delete;
463 SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete;
465 #ifdef KOKKOS_ENABLE_DEBUG 466 static RecordBase s_root_record;
469 const Kokkos::Experimental::HIPSpace m_space;
472 ~SharedAllocationRecord();
474 SharedAllocationRecord(
475 const Kokkos::Experimental::HIPSpace& arg_space,
476 const std::string& arg_label, const size_t arg_alloc_size,
477 const RecordBase::function_type arg_dealloc = &base_t::deallocate);
481 class SharedAllocationRecord<Kokkos::Experimental::HIPHostPinnedSpace, void>
482 :
public SharedAllocationRecordCommon<
483 Kokkos::Experimental::HIPHostPinnedSpace> {
485 friend class SharedAllocationRecordCommon<
486 Kokkos::Experimental::HIPHostPinnedSpace>;
488 SharedAllocationRecordCommon<Kokkos::Experimental::HIPHostPinnedSpace>;
489 using RecordBase = SharedAllocationRecord<void, void>;
491 SharedAllocationRecord(const SharedAllocationRecord&) = delete;
492 SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete;
494 #ifdef KOKKOS_ENABLE_DEBUG 495 static RecordBase s_root_record;
498 const Kokkos::Experimental::HIPHostPinnedSpace m_space;
501 ~SharedAllocationRecord();
502 SharedAllocationRecord() = default;
504 SharedAllocationRecord(
505 const Kokkos::Experimental::HIPHostPinnedSpace& arg_space,
506 const std::string& arg_label, const size_t arg_alloc_size,
507 const RecordBase::function_type arg_dealloc = &base_t::deallocate);
529 using execution_space = HIP;
530 using memory_space = HIPSpace;
531 using device_type = Kokkos::Device<execution_space, memory_space>;
533 using array_layout = LayoutLeft;
534 using size_type = HIPSpace::size_type;
536 using scratch_memory_space = ScratchMemorySpace<HIP>;
539 HIP(hipStream_t stream);
546 KOKKOS_INLINE_FUNCTION
static int in_parallel() {
547 #if defined(__HIP_ARCH__) 560 static void impl_static_fence();
564 hipStream_t hip_stream()
const;
567 static void print_configuration(std::ostream&,
const bool detail =
false);
570 static void impl_finalize();
575 struct SelectDevice {
577 SelectDevice() : hip_device_id(0) {}
578 explicit SelectDevice(
int id) : hip_device_id(id) {}
581 int hip_device()
const;
582 static hipDeviceProp_t
const& hip_device_prop();
584 static void impl_initialize(
const SelectDevice = SelectDevice());
586 static int impl_is_initialized();
590 static size_type detect_device_count();
592 static int concurrency();
593 static const char* name();
595 inline Impl::HIPInternal* impl_internal_space_instance()
const {
596 return m_space_instance.get();
599 uint32_t impl_instance_id() const noexcept {
return 0; }
602 Kokkos::Impl::HostSharedPtr<Impl::HIPInternal> m_space_instance;
608 struct DeviceTypeTraits<
Kokkos::Experimental::HIP> {
609 static constexpr DeviceType
id = DeviceType::HIP;
616 class HIPSpaceInitializer :
public Kokkos::Impl::ExecSpaceInitializerBase {
618 HIPSpaceInitializer() =
default;
619 ~HIPSpaceInitializer() =
default;
620 void initialize(
const InitArguments& args)
final;
621 void finalize(
const bool)
final;
623 void print_configuration(std::ostream& msg, const
bool detail) final;
633 struct MemorySpaceAccess<
Kokkos::Experimental::HIPSpace,
634 Kokkos::Experimental::HIP::scratch_memory_space> {
635 enum :
bool { assignable =
false };
636 enum :
bool { accessible =
true };
637 enum :
bool { deepcopy =
false };
Memory management for host memory.
Declaration of various MemoryLayout options.
Access relationship between DstMemorySpace and SrcMemorySpace.