//===----------- api.cpp - Target independent OpenMP target RTL -----------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Implementation of OpenMP API interface functions.
//
//===----------------------------------------------------------------------===//
#include "PluginManager.h"
#include "device.h"
#include "omptarget.h"
#include "rtl.h"
#include "OpenMP/InternalTypes.h"
#include "OpenMP/Mapping.h"
#include "OpenMP/OMPT/Interface.h"
#include "OpenMP/omp.h"
#include "Shared/Profile.h"
#include "llvm/ADT/SmallVector.h"
#include <climits>
#include <cstdlib>
#include <cstring>
#include <mutex>
EXTERN void ompx_dump_mapping_tables() {
ident_t Loc = {0, 0, 0, 0, ";libomptarget;libomptarget;0;0;;"};
auto ExclusiveDevicesAccessor = PM->getExclusiveDevicesAccessor();
for (auto &Device : PM->devices(ExclusiveDevicesAccessor))
dumpTargetPointerMappings(&Loc, Device, true);
}
#ifdef OMPT_SUPPORT
using namespace llvm::omp::target::ompt;
#endif
void *targetAllocExplicit(size_t Size, int DeviceNum, int Kind,
const char *Name);
void targetFreeExplicit(void *DevicePtr, int DeviceNum, int Kind,
const char *Name);
void *targetLockExplicit(void *HostPtr, size_t Size, int DeviceNum,
const char *Name);
void targetUnlockExplicit(void *HostPtr, int DeviceNum, const char *Name);
// Implemented in libomp, they are called from within __tgt_* functions.
extern "C" {
int __kmpc_get_target_offload(void) __attribute__((weak));
kmp_task_t *__kmpc_omp_task_alloc(ident_t *loc_ref, int32_t gtid, int32_t flags,
size_t sizeof_kmp_task_t,
size_t sizeof_shareds,
kmp_routine_entry_t task_entry)
__attribute__((weak));
kmp_task_t *
__kmpc_omp_target_task_alloc(ident_t *loc_ref, int32_t gtid, int32_t flags,
size_t sizeof_kmp_task_t, size_t sizeof_shareds,
kmp_routine_entry_t task_entry, int64_t device_id)
__attribute__((weak));
int32_t __kmpc_omp_task_with_deps(ident_t *loc_ref, int32_t gtid,
kmp_task_t *new_task, int32_t ndeps,
kmp_depend_info_t *dep_list,
int32_t ndeps_noalias,
kmp_depend_info_t *noalias_dep_list)
__attribute__((weak));
}
EXTERN int omp_get_num_devices(void) {
TIMESCOPE();
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
size_t NumDevices = PM->getNumDevices();
DP("Call to omp_get_num_devices returning %zd\n", NumDevices);
return NumDevices;
}
EXTERN int omp_get_device_num(void) {
TIMESCOPE();
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
int HostDevice = omp_get_initial_device();
DP("Call to omp_get_device_num returning %d\n", HostDevice);
return HostDevice;
}
EXTERN int omp_get_initial_device(void) {
TIMESCOPE();
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
int HostDevice = omp_get_num_devices();
DP("Call to omp_get_initial_device returning %d\n", HostDevice);
return HostDevice;
}
EXTERN void *omp_target_alloc(size_t Size, int DeviceNum) {
TIMESCOPE_WITH_DETAILS("dst_dev=" + std::to_string(DeviceNum) +
";size=" + std::to_string(Size));
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_DEFAULT, __func__);
}
EXTERN void *llvm_omp_target_alloc_device(size_t Size, int DeviceNum) {
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_DEVICE, __func__);
}
EXTERN void *llvm_omp_target_alloc_host(size_t Size, int DeviceNum) {
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_HOST, __func__);
}
EXTERN void *llvm_omp_target_alloc_shared(size_t Size, int DeviceNum) {
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_SHARED, __func__);
}
EXTERN void omp_target_free(void *Ptr, int DeviceNum) {
TIMESCOPE();
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
return targetFreeExplicit(Ptr, DeviceNum, TARGET_ALLOC_DEFAULT, __func__);
}
EXTERN void llvm_omp_target_free_device(void *Ptr, int DeviceNum) {
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
return targetFreeExplicit(Ptr, DeviceNum, TARGET_ALLOC_DEVICE, __func__);
}
EXTERN void llvm_omp_target_free_host(void *Ptr, int DeviceNum) {
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
return targetFreeExplicit(Ptr, DeviceNum, TARGET_ALLOC_HOST, __func__);
}
EXTERN void llvm_omp_target_free_shared(void *Ptre, int DeviceNum) {
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
return targetFreeExplicit(Ptre, DeviceNum, TARGET_ALLOC_SHARED, __func__);
}
EXTERN void *llvm_omp_target_dynamic_shared_alloc() {
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
return nullptr;
}
EXTERN void *llvm_omp_get_dynamic_shared() {
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
return nullptr;
}
EXTERN [[nodiscard]] void *llvm_omp_target_lock_mem(void *Ptr, size_t Size,
int DeviceNum) {
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
return targetLockExplicit(Ptr, Size, DeviceNum, __func__);
}
EXTERN void llvm_omp_target_unlock_mem(void *Ptr, int DeviceNum) {
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
targetUnlockExplicit(Ptr, DeviceNum, __func__);
}
EXTERN int omp_target_is_present(const void *Ptr, int DeviceNum) {
TIMESCOPE();
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n",
DeviceNum, DPxPTR(Ptr));
if (!Ptr) {
DP("Call to omp_target_is_present with NULL ptr, returning false\n");
return false;
}
if (DeviceNum == omp_get_initial_device()) {
DP("Call to omp_target_is_present on host, returning true\n");
return true;
}
auto DeviceOrErr = PM->getDevice(DeviceNum);
if (!DeviceOrErr)
FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());
// omp_target_is_present tests whether a host pointer refers to storage that
// is mapped to a given device. However, due to the lack of the storage size,
// only check 1 byte. Cannot set size 0 which checks whether the pointer (zero
// lengh array) is mapped instead of the referred storage.
TargetPointerResultTy TPR =
DeviceOrErr->getMappingInfo().getTgtPtrBegin(const_cast<void *>(Ptr), 1,
/*UpdateRefCount=*/false,
/*UseHoldRefCount=*/false);
int Rc = TPR.isPresent();
DP("Call to omp_target_is_present returns %d\n", Rc);
return Rc;
}
EXTERN int omp_target_memcpy(void *Dst, const void *Src, size_t Length,
size_t DstOffset, size_t SrcOffset, int DstDevice,
int SrcDevice) {
TIMESCOPE_WITH_DETAILS("dst_dev=" + std::to_string(DstDevice) +
";src_dev=" + std::to_string(SrcDevice) +
";size=" + std::to_string(Length));
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
DP("Call to omp_target_memcpy, dst device %d, src device %d, "
"dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, "
"src offset %zu, length %zu\n",
DstDevice, SrcDevice, DPxPTR(Dst), DPxPTR(Src), DstOffset, SrcOffset,
Length);
if (!Dst || !Src || Length <= 0) {
if (Length == 0) {
DP("Call to omp_target_memcpy with zero length, nothing to do\n");
return OFFLOAD_SUCCESS;
}
REPORT("Call to omp_target_memcpy with invalid arguments\n");
return OFFLOAD_FAIL;
}
int Rc = OFFLOAD_SUCCESS;
void *SrcAddr = (char *)const_cast<void *>(Src) + SrcOffset;
void *DstAddr = (char *)Dst + DstOffset;
if (SrcDevice == omp_get_initial_device() &&
DstDevice == omp_get_initial_device()) {
DP("copy from host to host\n");
const void *P = memcpy(DstAddr, SrcAddr, Length);
if (P == NULL)
Rc = OFFLOAD_FAIL;
} else if (SrcDevice == omp_get_initial_device()) {
DP("copy from host to device\n");
auto DstDeviceOrErr = PM->getDevice(DstDevice);
if (!DstDeviceOrErr)
FATAL_MESSAGE(DstDevice, "%s",
toString(DstDeviceOrErr.takeError()).c_str());
AsyncInfoTy AsyncInfo(*DstDeviceOrErr);
Rc = DstDeviceOrErr->submitData(DstAddr, SrcAddr, Length, AsyncInfo);
} else if (DstDevice == omp_get_initial_device()) {
DP("copy from device to host\n");
auto SrcDeviceOrErr = PM->getDevice(SrcDevice);
if (!SrcDeviceOrErr)
FATAL_MESSAGE(SrcDevice, "%s",
toString(SrcDeviceOrErr.takeError()).c_str());
AsyncInfoTy AsyncInfo(*SrcDeviceOrErr);
Rc = SrcDeviceOrErr->retrieveData(DstAddr, SrcAddr, Length, AsyncInfo);
} else {
DP("copy from device to device\n");
auto SrcDeviceOrErr = PM->getDevice(SrcDevice);
if (!SrcDeviceOrErr)
FATAL_MESSAGE(SrcDevice, "%s",
toString(SrcDeviceOrErr.takeError()).c_str());
AsyncInfoTy AsyncInfo(*SrcDeviceOrErr);
auto DstDeviceOrErr = PM->getDevice(DstDevice);
if (!DstDeviceOrErr)
FATAL_MESSAGE(DstDevice, "%s",
toString(DstDeviceOrErr.takeError()).c_str());
// First try to use D2D memcpy which is more efficient. If fails, fall back
// to unefficient way.
if (SrcDeviceOrErr->isDataExchangable(*DstDeviceOrErr)) {
AsyncInfoTy AsyncInfo(*SrcDeviceOrErr);
Rc = SrcDeviceOrErr->dataExchange(SrcAddr, *DstDeviceOrErr, DstAddr,
Length, AsyncInfo);
if (Rc == OFFLOAD_SUCCESS)
return OFFLOAD_SUCCESS;
}
void *Buffer = malloc(Length);
{
AsyncInfoTy AsyncInfo(*SrcDeviceOrErr);
Rc = SrcDeviceOrErr->retrieveData(Buffer, SrcAddr, Length, AsyncInfo);
}
if (Rc == OFFLOAD_SUCCESS) {
AsyncInfoTy AsyncInfo(*DstDeviceOrErr);
Rc = DstDeviceOrErr->submitData(DstAddr, Buffer, Length, AsyncInfo);
}
free(Buffer);
}
DP("omp_target_memcpy returns %d\n", Rc);
return Rc;
}
// The helper function that calls omp_target_memcpy or omp_target_memcpy_rect
static int libomp_target_memcpy_async_task(int32_t Gtid, kmp_task_t *Task) {
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
if (Task == nullptr)
return OFFLOAD_FAIL;
TargetMemcpyArgsTy *Args = (TargetMemcpyArgsTy *)Task->shareds;
if (Args == nullptr)
return OFFLOAD_FAIL;
// Call blocked version
int Rc = OFFLOAD_SUCCESS;
if (Args->IsRectMemcpy) {
Rc = omp_target_memcpy_rect(
Args->Dst, Args->Src, Args->ElementSize, Args->NumDims, Args->Volume,
Args->DstOffsets, Args->SrcOffsets, Args->DstDimensions,
Args->SrcDimensions, Args->DstDevice, Args->SrcDevice);
DP("omp_target_memcpy_rect returns %d\n", Rc);
} else {
Rc = omp_target_memcpy(Args->Dst, Args->Src, Args->Length, Args->DstOffset,
Args->SrcOffset, Args->DstDevice, Args->SrcDevice);
DP("omp_target_memcpy returns %d\n", Rc);
}
// Release the arguments object
delete Args;
return Rc;
}
static int libomp_target_memset_async_task(int32_t Gtid, kmp_task_t *Task) {
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
if (!Task)
return OFFLOAD_FAIL;
auto *Args = reinterpret_cast<TargetMemsetArgsTy *>(Task->shareds);
if (!Args)
return OFFLOAD_FAIL;
// call omp_target_memset()
omp_target_memset(Args->Ptr, Args->C, Args->N, Args->DeviceNum);
delete Args;
return OFFLOAD_SUCCESS;
}
static inline void
convertDepObjVector(llvm::SmallVector<kmp_depend_info_t> &Vec, int DepObjCount,
omp_depend_t *DepObjList) {
for (int i = 0; i < DepObjCount; ++i) {
omp_depend_t DepObj = DepObjList[i];
Vec.push_back(*((kmp_depend_info_t *)DepObj));
}
}
template <class T>
static inline int
libomp_helper_task_creation(T *Args, int (*Fn)(int32_t, kmp_task_t *),
int DepObjCount, omp_depend_t *DepObjList) {
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
// Create global thread ID
int Gtid = __kmpc_global_thread_num(nullptr);
// Setup the hidden helper flags
int32_t Flags = 0;
kmp_tasking_flags_t *InputFlags = (kmp_tasking_flags_t *)&Flags;
InputFlags->hidden_helper = 1;
// Alloc the helper task
kmp_task_t *Task = __kmpc_omp_target_task_alloc(
nullptr, Gtid, Flags, sizeof(kmp_task_t), 0, Fn, -1);
if (!Task) {
delete Args;
return OFFLOAD_FAIL;
}
// Setup the arguments for the helper task
Task->shareds = Args;
// Convert types of depend objects
llvm::SmallVector<kmp_depend_info_t> DepObjs;
convertDepObjVector(DepObjs, DepObjCount, DepObjList);
// Launch the helper task
int Rc = __kmpc_omp_task_with_deps(nullptr, Gtid, Task, DepObjCount,
DepObjs.data(), 0, nullptr);
return Rc;
}
EXTERN void *omp_target_memset(void *Ptr, int ByteVal, size_t NumBytes,
int DeviceNum) {
TIMESCOPE();
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
DP("Call to omp_target_memset, device %d, device pointer %p, size %zu\n",
DeviceNum, Ptr, NumBytes);
// Behave as a no-op if N==0 or if Ptr is nullptr (as a useful implementation
// of unspecified behavior, see OpenMP spec).
if (!Ptr || NumBytes == 0) {
return Ptr;
}
if (DeviceNum == omp_get_initial_device()) {
DP("filling memory on host via memset");
memset(Ptr, ByteVal, NumBytes); // ignore return value, memset() cannot fail
} else {
// TODO: replace the omp_target_memset() slow path with the fast path.
// That will require the ability to execute a kernel from within
// libomptarget.so (which we do not have at the moment).
// This is a very slow path: create a filled array on the host and upload
// it to the GPU device.
int InitialDevice = omp_get_initial_device();
void *Shadow = omp_target_alloc(NumBytes, InitialDevice);
if (Shadow) {
(void)memset(Shadow, ByteVal, NumBytes);
(void)omp_target_memcpy(Ptr, Shadow, NumBytes, 0, 0, DeviceNum,
InitialDevice);
(void)omp_target_free(Shadow, InitialDevice);
} else {
// If the omp_target_alloc has failed, let's just not do anything.
// omp_target_memset does not have any good way to fail, so we
// simply avoid a catastrophic failure of the process for now.
DP("omp_target_memset failed to fill memory due to error with "
"omp_target_alloc");
}
}
DP("omp_target_memset returns %p\n", Ptr);
return Ptr;
}
EXTERN void *omp_target_memset_async(void *Ptr, int ByteVal, size_t NumBytes,
int DeviceNum, int DepObjCount,
omp_depend_t *DepObjList) {
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
DP("Call to omp_target_memset_async, device %d, device pointer %p, size %zu",
DeviceNum, Ptr, NumBytes);
// Behave as a no-op if N==0 or if Ptr is nullptr (as a useful implementation
// of unspecified behavior, see OpenMP spec).
if (!Ptr || NumBytes == 0)
return Ptr;
// Create the task object to deal with the async invocation
auto *Args = new TargetMemsetArgsTy{Ptr, ByteVal, NumBytes, DeviceNum};
// omp_target_memset_async() cannot fail via a return code, so ignore the
// return code of the helper function
(void)libomp_helper_task_creation(Args, &libomp_target_memset_async_task,
DepObjCount, DepObjList);
return Ptr;
}
EXTERN int omp_target_memcpy_async(void *Dst, const void *Src, size_t Length,
size_t DstOffset, size_t SrcOffset,
int DstDevice, int SrcDevice,
int DepObjCount, omp_depend_t *DepObjList) {
TIMESCOPE_WITH_DETAILS("dst_dev=" + std::to_string(DstDevice) +
";src_dev=" + std::to_string(SrcDevice) +
";size=" + std::to_string(Length));
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
DP("Call to omp_target_memcpy_async, dst device %d, src device %d, "
"dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, "
"src offset %zu, length %zu\n",
DstDevice, SrcDevice, DPxPTR(Dst), DPxPTR(Src), DstOffset, SrcOffset,
Length);
// Check the source and dest address
if (Dst == nullptr || Src == nullptr)
return OFFLOAD_FAIL;
// Create task object
TargetMemcpyArgsTy *Args = new TargetMemcpyArgsTy(
Dst, Src, Length, DstOffset, SrcOffset, DstDevice, SrcDevice);
// Create and launch helper task
int Rc = libomp_helper_task_creation(Args, &libomp_target_memcpy_async_task,
DepObjCount, DepObjList);
DP("omp_target_memcpy_async returns %d\n", Rc);
return Rc;
}
EXTERN int
omp_target_memcpy_rect(void *Dst, const void *Src, size_t ElementSize,
int NumDims, const size_t *Volume,
const size_t *DstOffsets, const size_t *SrcOffsets,
const size_t *DstDimensions, const size_t *SrcDimensions,
int DstDevice, int SrcDevice) {
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, "
"dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", "
"src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", "
"volume " DPxMOD ", element size %zu, num_dims %d\n",
DstDevice, SrcDevice, DPxPTR(Dst), DPxPTR(Src), DPxPTR(DstOffsets),
DPxPTR(SrcOffsets), DPxPTR(DstDimensions), DPxPTR(SrcDimensions),
DPxPTR(Volume), ElementSize, NumDims);
if (!(Dst || Src)) {
DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n",
INT_MAX);
return INT_MAX;
}
if (!Dst || !Src || ElementSize < 1 || NumDims < 1 || !Volume ||
!DstOffsets || !SrcOffsets || !DstDimensions || !SrcDimensions) {
REPORT("Call to omp_target_memcpy_rect with invalid arguments\n");
return OFFLOAD_FAIL;
}
int Rc;
if (NumDims == 1) {
Rc = omp_target_memcpy(Dst, Src, ElementSize * Volume[0],
ElementSize * DstOffsets[0],
ElementSize * SrcOffsets[0], DstDevice, SrcDevice);
} else {
size_t DstSliceSize = ElementSize;
size_t SrcSliceSize = ElementSize;
for (int I = 1; I < NumDims; ++I) {
DstSliceSize *= DstDimensions[I];
SrcSliceSize *= SrcDimensions[I];
}
size_t DstOff = DstOffsets[0] * DstSliceSize;
size_t SrcOff = SrcOffsets[0] * SrcSliceSize;
for (size_t I = 0; I < Volume[0]; ++I) {
Rc = omp_target_memcpy_rect(
(char *)Dst + DstOff + DstSliceSize * I,
(char *)const_cast<void *>(Src) + SrcOff + SrcSliceSize * I,
ElementSize, NumDims - 1, Volume + 1, DstOffsets + 1, SrcOffsets + 1,
DstDimensions + 1, SrcDimensions + 1, DstDevice, SrcDevice);
if (Rc) {
DP("Recursive call to omp_target_memcpy_rect returns unsuccessfully\n");
return Rc;
}
}
}
DP("omp_target_memcpy_rect returns %d\n", Rc);
return Rc;
}
EXTERN int omp_target_memcpy_rect_async(
void *Dst, const void *Src, size_t ElementSize, int NumDims,
const size_t *Volume, const size_t *DstOffsets, const size_t *SrcOffsets,
const size_t *DstDimensions, const size_t *SrcDimensions, int DstDevice,
int SrcDevice, int DepObjCount, omp_depend_t *DepObjList) {
TIMESCOPE_WITH_DETAILS("dst_dev=" + std::to_string(DstDevice) +
";src_dev=" + std::to_string(SrcDevice) +
";size=" + std::to_string(ElementSize) +
";num_dims=" + std::to_string(NumDims));
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
DP("Call to omp_target_memcpy_rect_async, dst device %d, src device %d, "
"dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", "
"src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", "
"volume " DPxMOD ", element size %zu, num_dims %d\n",
DstDevice, SrcDevice, DPxPTR(Dst), DPxPTR(Src), DPxPTR(DstOffsets),
DPxPTR(SrcOffsets), DPxPTR(DstDimensions), DPxPTR(SrcDimensions),
DPxPTR(Volume), ElementSize, NumDims);
// Need to check this first to not return OFFLOAD_FAIL instead
if (!Dst && !Src) {
DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n",
INT_MAX);
return INT_MAX;
}
// Check the source and dest address
if (Dst == nullptr || Src == nullptr)
return OFFLOAD_FAIL;
// Create task object
TargetMemcpyArgsTy *Args = new TargetMemcpyArgsTy(
Dst, Src, ElementSize, NumDims, Volume, DstOffsets, SrcOffsets,
DstDimensions, SrcDimensions, DstDevice, SrcDevice);
// Create and launch helper task
int Rc = libomp_helper_task_creation(Args, &libomp_target_memcpy_async_task,
DepObjCount, DepObjList);
DP("omp_target_memcpy_rect_async returns %d\n", Rc);
return Rc;
}
EXTERN int omp_target_associate_ptr(const void *HostPtr, const void *DevicePtr,
size_t Size, size_t DeviceOffset,
int DeviceNum) {
TIMESCOPE();
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", "
"device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n",
DPxPTR(HostPtr), DPxPTR(DevicePtr), Size, DeviceOffset, DeviceNum);
if (!HostPtr || !DevicePtr || Size <= 0) {
REPORT("Call to omp_target_associate_ptr with invalid arguments\n");
return OFFLOAD_FAIL;
}
if (DeviceNum == omp_get_initial_device()) {
REPORT("omp_target_associate_ptr: no association possible on the host\n");
return OFFLOAD_FAIL;
}
auto DeviceOrErr = PM->getDevice(DeviceNum);
if (!DeviceOrErr)
FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());
void *DeviceAddr = (void *)((uint64_t)DevicePtr + (uint64_t)DeviceOffset);
OMPT_IF_BUILT(InterfaceRAII(
RegionInterface.getCallbacks<ompt_target_data_associate>(), DeviceNum,
const_cast<void *>(HostPtr), const_cast<void *>(DevicePtr), Size,
__builtin_return_address(0)));
int Rc = DeviceOrErr->getMappingInfo().associatePtr(
const_cast<void *>(HostPtr), const_cast<void *>(DeviceAddr), Size);
DP("omp_target_associate_ptr returns %d\n", Rc);
return Rc;
}
EXTERN int omp_target_disassociate_ptr(const void *HostPtr, int DeviceNum) {
TIMESCOPE();
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", "
"device_num %d\n",
DPxPTR(HostPtr), DeviceNum);
if (!HostPtr) {
REPORT("Call to omp_target_associate_ptr with invalid host_ptr\n");
return OFFLOAD_FAIL;
}
if (DeviceNum == omp_get_initial_device()) {
REPORT(
"omp_target_disassociate_ptr: no association possible on the host\n");
return OFFLOAD_FAIL;
}
auto DeviceOrErr = PM->getDevice(DeviceNum);
if (!DeviceOrErr)
FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());
OMPT_IF_BUILT(InterfaceRAII(
RegionInterface.getCallbacks<ompt_target_data_disassociate>(), DeviceNum,
const_cast<void *>(HostPtr),
/*DevicePtr=*/nullptr, /*Size=*/0, __builtin_return_address(0)));
int Rc = DeviceOrErr->getMappingInfo().disassociatePtr(
const_cast<void *>(HostPtr));
DP("omp_target_disassociate_ptr returns %d\n", Rc);
return Rc;
}
EXTERN void *omp_get_mapped_ptr(const void *Ptr, int DeviceNum) {
TIMESCOPE();
OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));
DP("Call to omp_get_mapped_ptr with ptr " DPxMOD ", device_num %d.\n",
DPxPTR(Ptr), DeviceNum);
if (!Ptr) {
REPORT("Call to omp_get_mapped_ptr with nullptr.\n");
return nullptr;
}
int NumDevices = omp_get_initial_device();
if (DeviceNum == NumDevices) {
DP("Device %d is initial device, returning Ptr " DPxMOD ".\n",
DeviceNum, DPxPTR(Ptr));
return const_cast<void *>(Ptr);
}
if (NumDevices <= DeviceNum) {
DP("DeviceNum %d is invalid, returning nullptr.\n", DeviceNum);
return nullptr;
}
auto DeviceOrErr = PM->getDevice(DeviceNum);
if (!DeviceOrErr)
FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());
TargetPointerResultTy TPR =
DeviceOrErr->getMappingInfo().getTgtPtrBegin(const_cast<void *>(Ptr), 1,
/*UpdateRefCount=*/false,
/*UseHoldRefCount=*/false);
if (!TPR.isPresent()) {
DP("Ptr " DPxMOD "is not present on device %d, returning nullptr.\n",
DPxPTR(Ptr), DeviceNum);
return nullptr;
}
DP("omp_get_mapped_ptr returns " DPxMOD ".\n", DPxPTR(TPR.TargetPointer));
return TPR.TargetPointer;
}
Codebase Browser
llvm