memory.c | Explore in Territory

// Copyright 2021 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.

// Include first for the platform detection macros.
#include "xnnpack/common.h"

#if XNN_PLATFORM_WEB
#include <emscripten/emscripten.h>
#endif  // XNN_PLATFORM_WEB

#if XNN_PLATFORM_WINDOWS
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif

#include <inttypes.h>
#include <windows.h>

#else
// This define needs to come first because errno include features.h and would have defined macros that lead to
// sys/mman.h not having mremap.
#if !defined(_GNU_SOURCE)
#define _GNU_SOURCE
#endif

#if XNN_ARCH_ARM && XNN_PLATFORM_JIT
#include <sys/syscall.h>
#endif

#include <errno.h>
#include <sys/mman.h>
#include <unistd.h>
#endif  // XNN_PLATFORM_WINDOWS

#if XNN_PLATFORM_QURT
#include "qurt/qurt_alloc.h"  // NOLINT - header provided by Hexagon toolchain
#endif

#include <assert.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>

#include "xnnpack.h"
#include "xnnpack/log.h"
#include "xnnpack/math.h"
#include "xnnpack/memory.h"

// Helpers to allocate/mmap and release memory used by both code and weights cache.

static size_t system_page_size = …;

static size_t get_page_size() { … }

// Maps `size` bytes of memory, returns pointer to allocation, NULL if failed.
static void* allocate_buffer(size_t size) { … }

// Releases memory previously mapped by `allocate_buffer`, returns xnn_status_success on success.
static enum xnn_status release_memory(void* start, size_t capacity) { … }

// Resize a buffer at old_pointer of size old_bytes to new_size. The actual new size of the resized buffer is written to
// new_capacity_out, which can be >= new_size due to page alignment requirements.
// Returns a pointer to a buffer which might be the same as old_pointer if we can remap virtual memory, otherwise we
// allocate a new buffer and copy contents of old_buffer over.
static void* resize_buffer(
  void* old_pointer, size_t old_size, size_t old_capacity, size_t new_size, size_t* new_capacity_out)
{ … }

enum xnn_status xnn_allocate_code_memory(struct xnn_code_buffer* buffer, size_t size) { … }

// Releases unused memory. Will write the new capacity to `capacity`.
static enum xnn_status release_unused_memory(size_t size, void* start, size_t* capacity) { … }

enum xnn_memory_permission { … };

static enum xnn_status set_memory_permission(void* start, size_t size, enum xnn_memory_permission permission) { … }

#if XNN_PLATFORM_WEB
EM_JS(int, xnnLoadWasmModuleJS, (const uint8_t* code, int offset, int offset_end, int invalid_function_index), {
    const tableOriginalSize = wasmTable.length;
    const binary = new Uint8Array(HEAPU8.slice(code + offset, code + offset_end));
    try {
      var module = new WebAssembly.Module(binary);
      var instance = new WebAssembly.Instance(module, {env : {memory: wasmMemory}});
      for (var symName in instance.exports) {
        var value = instance.exports[symName];
        addFunction(value);
      }
      if (tableOriginalSize < wasmTable.length) {
        return tableOriginalSize;
      }
      return invalid_function_index;
    }
    catch(error) {
      console.log(error);
      return invalid_function_index;
    }
});
#endif // XNN_PLATFORM_WEB

#if XNN_PLATFORM_JIT
enum xnn_status xnn_finalize_code_memory(struct xnn_code_buffer* buffer) {
  #if XNN_PLATFORM_WEB
    return xnn_status_success;
  #else
    const enum xnn_status status = release_unused_memory(buffer->size, buffer->start, &buffer->capacity);
    if (status != xnn_status_success) {
      return status;
    }

    if (buffer->capacity == 0) {
      return xnn_status_success;
    }

    // Flush icache, do it before changing permissions due to bugs on older ARM64 kernels.
    #if (XNN_ARCH_ARM || XNN_ARCH_ARM64) && XNN_PLATFORM_JIT
      #if XNN_PLATFORM_WINDOWS
        FlushInstructionCache(GetCurrentProcess(), buffer->start, buffer->capacity);
      #else
        #if XNN_ARCH_ARM
          // TODO(b/309410154): Re-enable once __ARM_NR_cacheflush is fixed
          // why, but something broke with cl/576649879. See b/307871190.
          syscall(__ARM_NR_cacheflush, buffer->start, (void*) ((uint8_t*) buffer->start + buffer->capacity), 0);
        #else
        // iOS toolchain doesn't support this, use sys_icache_invalidate, when we support iOS.
        __builtin___clear_cache(buffer->start, (void*) ((uint8_t*) buffer->start + buffer->capacity));
        #endif  // XNN_ARCH_ARM
      #endif  // XNN_PLATFORM_WINDOWS
    #endif  // (XNN_ARCH_ARM || XNN_ARCH_ARM64) && !XNN_PLATFORM_IOS

    // Set permissions to RX (no write).
    return set_memory_permission(buffer->start, buffer->size, xnn_memory_permission_read_execute);
  #endif // XNN_PLATFORM_WEB
}

uintptr_t xnn_first_function_in_chunk_ptr(struct xnn_code_buffer* buffer, size_t offset, size_t offset_end) {
  #if (XNN_ARCH_ARM || XNN_ARCH_ARM64)
    return (uintptr_t) buffer->start + offset;
  #elif XNN_PLATFORM_WEB
    if (offset == offset_end) {
      return XNN_INVALID_FUNCTION_INDEX;
    }
    return xnnLoadWasmModuleJS(buffer->start, offset, offset_end, XNN_INVALID_FUNCTION_INDEX);
  #endif
}
#endif  // XNN_PLATFORM_JIT

enum xnn_status xnn_release_code_memory(struct xnn_code_buffer* buffer) { … }

enum xnn_status xnn_reserve_code_memory(struct xnn_code_buffer* buffer, size_t min_available_size) { … }

enum xnn_status xnn_allocate_weights_memory(struct xnn_weights_buffer* buffer, size_t size) { … }

enum xnn_status xnn_release_weights_memory(struct xnn_weights_buffer* buffer) { … }

enum xnn_status xnn_reserve_weights_memory(struct xnn_weights_buffer* buffer, size_t min_available_size) { … }

enum xnn_status xnn_finalize_weights_memory(struct xnn_weights_buffer* buffer) { … }
chromium/third_party/xnnpack/src/src/memory.c
