godot/thirdparty/embree/common/sys/alloc.cpp

// Copyright 2009-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0

#include "alloc.h"
#include "intrinsics.h"
#include "sysinfo.h"
#include "mutex.h"

////////////////////////////////////////////////////////////////////////////////
/// All Platforms
////////////////////////////////////////////////////////////////////////////////
  
namespace embree
{
  size_t total_allocations = …;

#if defined(EMBREE_SYCL_SUPPORT)
  
  __thread sycl::context* tls_context_tutorial = nullptr;
  __thread sycl::device* tls_device_tutorial = nullptr;
  
  __thread sycl::context* tls_context_embree = nullptr;
  __thread sycl::device* tls_device_embree = nullptr;
  
  void enableUSMAllocEmbree(sycl::context* context, sycl::device* device)
  {
    // -- GODOT start --
    // if (tls_context_embree != nullptr) throw std::runtime_error("USM allocation already enabled");
    // if (tls_device_embree != nullptr) throw std::runtime_error("USM allocation already enabled");
    if (tls_context_embree != nullptr) {
      abort();
    }
    if (tls_device_embree != nullptr) {
      abort();
    }
    // -- GODOT end --
    tls_context_embree = context;
    tls_device_embree = device;
  }

  void disableUSMAllocEmbree()
  {
    // -- GODOT start --
    // if (tls_context_embree  == nullptr) throw std::runtime_error("USM allocation not enabled");
    // if (tls_device_embree  == nullptr) throw std::runtime_error("USM allocation not enabled");
    if (tls_context_embree  == nullptr) {
      abort();
    }
    if (tls_device_embree  == nullptr) {
      abort();
    }
    // -- GODOT end --
    tls_context_embree = nullptr;
    tls_device_embree = nullptr;
  }

  void enableUSMAllocTutorial(sycl::context* context, sycl::device* device)
  {
    //if (tls_context_tutorial != nullptr) throw std::runtime_error("USM allocation already enabled");
    //if (tls_device_tutorial != nullptr) throw std::runtime_error("USM allocation already enabled");
    tls_context_tutorial = context;
    tls_device_tutorial = device;
  }

  void disableUSMAllocTutorial()
  {
    // -- GODOT start --
    // if (tls_context_tutorial  == nullptr) throw std::runtime_error("USM allocation not enabled");
    // if (tls_device_tutorial  == nullptr) throw std::runtime_error("USM allocation not enabled");
    if (tls_context_tutorial  == nullptr) {
      abort();
    }
    if (tls_device_tutorial  == nullptr) {
      abort();
    }
    // -- GODOT end --
    
    tls_context_tutorial = nullptr;
    tls_device_tutorial = nullptr;
  }

#endif
  
  void* alignedMalloc(size_t size, size_t align)
  { … }

  void alignedFree(void* ptr)
  { … }

#if defined(EMBREE_SYCL_SUPPORT)
  
  void* alignedSYCLMalloc(sycl::context* context, sycl::device* device, size_t size, size_t align, EmbreeUSMMode mode)
  {
    assert(context);
    assert(device);
    
    if (size == 0)
      return nullptr;

    assert((align & (align-1)) == 0);
    total_allocations++;    

    void* ptr = nullptr;
    if (mode == EMBREE_USM_SHARED_DEVICE_READ_ONLY)
      ptr = sycl::aligned_alloc_shared(align,size,*device,*context,sycl::ext::oneapi::property::usm::device_read_only());
    else
      ptr = sycl::aligned_alloc_shared(align,size,*device,*context);
      
    // -- GODOT start --
    // if (size != 0 && ptr == nullptr)
    //   throw std::bad_alloc();
    if (size != 0 && ptr == nullptr) {
      abort();
    }
    // -- GODOT end --

    return ptr;
  }
  
  static MutexSys g_alloc_mutex;
  
  void* alignedSYCLMalloc(size_t size, size_t align, EmbreeUSMMode mode)
  {
    if (tls_context_tutorial) return alignedSYCLMalloc(tls_context_tutorial, tls_device_tutorial, size, align, mode);
    if (tls_context_embree  ) return alignedSYCLMalloc(tls_context_embree,   tls_device_embree,   size, align, mode);
    return nullptr;
  }

  void alignedSYCLFree(sycl::context* context, void* ptr)
  {
    assert(context);
    if (ptr) {
      sycl::free(ptr,*context);
    }
  }

  void alignedSYCLFree(void* ptr)
  {
    if (tls_context_tutorial) return alignedSYCLFree(tls_context_tutorial, ptr);
    if (tls_context_embree  ) return alignedSYCLFree(tls_context_embree, ptr);
  }

#endif

  void* alignedUSMMalloc(size_t size, size_t align, EmbreeUSMMode mode)
  { … }

  void alignedUSMFree(void* ptr)
  { … }

  static bool huge_pages_enabled = …;
  static MutexSys os_init_mutex;

  __forceinline bool isHugePageCandidate(const size_t bytes)
  { … }
}

////////////////////////////////////////////////////////////////////////////////
/// Windows Platform
////////////////////////////////////////////////////////////////////////////////

#ifdef _WIN32

#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include <malloc.h>

namespace embree
{
  bool win_enable_selockmemoryprivilege (bool verbose)
  {
    HANDLE hToken;
    if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY | TOKEN_ADJUST_PRIVILEGES, &hToken)) {
      if (verbose) std::cout << "WARNING: OpenProcessToken failed while trying to enable SeLockMemoryPrivilege: " << GetLastError() << std::endl;
      return false;
    }

    TOKEN_PRIVILEGES tp;
    tp.PrivilegeCount = 1;
    tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;

    if (!LookupPrivilegeValueW(nullptr, L"SeLockMemoryPrivilege", &tp.Privileges[0].Luid)) {
      if (verbose) std::cout << "WARNING: LookupPrivilegeValue failed while trying to enable SeLockMemoryPrivilege: " << GetLastError() << std::endl;
      return false;
    }
    
    SetLastError(ERROR_SUCCESS);
    if (!AdjustTokenPrivileges(hToken, FALSE, &tp, sizeof(tp), nullptr, 0)) {
      if (verbose) std::cout << "WARNING: AdjustTokenPrivileges failed while trying to enable SeLockMemoryPrivilege" << std::endl;
      return false;
    }
    
    if (GetLastError() == ERROR_NOT_ALL_ASSIGNED) {
      if (verbose) std::cout << "WARNING: AdjustTokenPrivileges failed to enable SeLockMemoryPrivilege: Add SeLockMemoryPrivilege for current user and run process in elevated mode (Run as administrator)." << std::endl;
      return false;
    } 

    return true;
  }

  bool os_init(bool hugepages, bool verbose) 
  {
    Lock<MutexSys> lock(os_init_mutex);

    if (!hugepages) {
      huge_pages_enabled = false;
      return true;
    }

    if (GetLargePageMinimum() != PAGE_SIZE_2M) {
      huge_pages_enabled = false;
      return false;
    }

    huge_pages_enabled = true;
    return true;
  }

  void* os_malloc(size_t bytes, bool& hugepages)
  {
    if (bytes == 0) {
      hugepages = false;
      return nullptr;
    }

    /* try direct huge page allocation first */
    if (isHugePageCandidate(bytes)) 
    {
      int flags = MEM_COMMIT | MEM_RESERVE | MEM_LARGE_PAGES;
      char* ptr = (char*) VirtualAlloc(nullptr,bytes,flags,PAGE_READWRITE);
      if (ptr != nullptr) {
        hugepages = true;
        return ptr;
      }
    } 

    /* fall back to 4k pages */
    int flags = MEM_COMMIT | MEM_RESERVE;
    char* ptr = (char*) VirtualAlloc(nullptr,bytes,flags,PAGE_READWRITE);
    // -- GODOT start --
    // if (ptr == nullptr) throw std::bad_alloc();
    if (ptr == nullptr) {
      abort();
    }
    // -- GODOT end --
    hugepages = false;
    return ptr;
  }

  size_t os_shrink(void* ptr, size_t bytesNew, size_t bytesOld, bool hugepages) 
  {
    if (hugepages) // decommitting huge pages seems not to work under Windows
      return bytesOld;

    const size_t pageSize = hugepages ? PAGE_SIZE_2M : PAGE_SIZE_4K;
    bytesNew = (bytesNew+pageSize-1) & ~(pageSize-1);
    bytesOld = (bytesOld+pageSize-1) & ~(pageSize-1);
    if (bytesNew >= bytesOld)
      return bytesOld;

    // -- GODOT start --
    // if (!VirtualFree((char*)ptr+bytesNew,bytesOld-bytesNew,MEM_DECOMMIT))
    //   throw std::bad_alloc();
    if (!VirtualFree((char*)ptr+bytesNew,bytesOld-bytesNew,MEM_DECOMMIT)) {
      abort();
    }
    // -- GODOT end --

    return bytesNew;
  }

  void os_free(void* ptr, size_t bytes, bool hugepages) 
  {
    if (bytes == 0) 
      return;

    // -- GODOT start --
    // if (!VirtualFree(ptr,0,MEM_RELEASE))
    //   throw std::bad_alloc();
    if (!VirtualFree(ptr,0,MEM_RELEASE)) {
      abort();
    }
    // -- GODOT end --
  }

  void os_advise(void *ptr, size_t bytes)
  {
  }
}

#endif

////////////////////////////////////////////////////////////////////////////////
/// Unix Platform
////////////////////////////////////////////////////////////////////////////////

#if defined(__UNIX__)

#include <sys/mman.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <sstream>

#if defined(__MACOSX__)
#include <mach/vm_statistics.h>
#endif

namespace embree
{
  bool os_init(bool hugepages, bool verbose) 
  { … }

  void* os_malloc(size_t bytes, bool& hugepages)
  { … }

  size_t os_shrink(void* ptr, size_t bytesNew, size_t bytesOld, bool hugepages) 
  { … }

  void os_free(void* ptr, size_t bytes, bool hugepages) 
  { … }

  /* hint for transparent huge pages (THP) */
  void os_advise(void* pptr, size_t bytes)
  { … }
}

#endif