/* / Author: Sam Rushing <[email protected]> / Hacked for Unix by AMK / $Id$ / Modified to support mmap with offset - to map a 'window' of a file / Author: Yotam Medini [email protected] / / mmapmodule.cpp -- map a view of a file into memory / / todo: need permission flags, perhaps a 'chsize' analog / not all functions check range yet!!! / / / This version of mmapmodule.c has been changed significantly / from the original mmapfile.c on which it was based. / The original version of mmapfile is maintained by Sam at / ftp://squirl.nightmare.com/pub/python/python-ext. */ #ifndef Py_BUILD_CORE_BUILTIN #define Py_BUILD_CORE_MODULE … #endif #include <Python.h> #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_fileutils.h" // _Py_stat_struct #include <stddef.h> // offsetof() #ifndef MS_WINDOWS # include <unistd.h> // close() #endif #ifndef MS_WINDOWS #define UNIX # ifdef HAVE_FCNTL_H # include <fcntl.h> # endif /* HAVE_FCNTL_H */ #endif #ifdef MS_WINDOWS #include <windows.h> #include <ntsecapi.h> // LsaNtStatusToWinError static int my_getpagesize(void) { SYSTEM_INFO si; GetSystemInfo(&si); return si.dwPageSize; } static int my_getallocationgranularity (void) { SYSTEM_INFO si; GetSystemInfo(&si); return si.dwAllocationGranularity; } #endif #ifdef UNIX #include <sys/mman.h> #include <sys/stat.h> #if defined(HAVE_SYSCONF) && defined(_SC_PAGESIZE) static int my_getpagesize(void) { … } #define my_getallocationgranularity … #else #define my_getpagesize … #endif #endif /* UNIX */ #include <string.h> #ifdef HAVE_SYS_TYPES_H #include <sys/types.h> #endif /* HAVE_SYS_TYPES_H */ /* Prefer MAP_ANONYMOUS since MAP_ANON is deprecated according to man page. */ #if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) #define MAP_ANONYMOUS … #endif access_mode; mmap_object; static int mmap_object_traverse(mmap_object *m_obj, visitproc visit, void *arg) { … } static void mmap_object_dealloc(mmap_object *m_obj) { … } static PyObject * mmap_close_method(mmap_object *self, PyObject *Py_UNUSED(ignored)) { … } #ifdef MS_WINDOWS #define CHECK_VALID … #define CHECK_VALID_OR_RELEASE … #endif /* MS_WINDOWS */ #ifdef UNIX #define CHECK_VALID(err) … #define CHECK_VALID_OR_RELEASE(err, buffer) … #endif /* UNIX */ #if defined(MS_WINDOWS) && !defined(DONT_USE_SEH) static DWORD filter_page_exception(EXCEPTION_POINTERS *ptrs, EXCEPTION_RECORD *record) { *record = *ptrs->ExceptionRecord; if (record->ExceptionCode == EXCEPTION_IN_PAGE_ERROR || record->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { return EXCEPTION_EXECUTE_HANDLER; } return EXCEPTION_CONTINUE_SEARCH; } static DWORD filter_page_exception_method(mmap_object *self, EXCEPTION_POINTERS *ptrs, EXCEPTION_RECORD *record) { *record = *ptrs->ExceptionRecord; if (record->ExceptionCode == EXCEPTION_IN_PAGE_ERROR || record->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { ULONG_PTR address = record->ExceptionInformation[1]; if (address >= (ULONG_PTR) self->data && address < (ULONG_PTR) self->data + (ULONG_PTR) self->size) { return EXCEPTION_EXECUTE_HANDLER; } } return EXCEPTION_CONTINUE_SEARCH; } #endif #if defined(MS_WINDOWS) && !defined(DONT_USE_SEH) #define HANDLE_INVALID_MEM … #else #define HANDLE_INVALID_MEM(sourcecode) … #endif #if defined(MS_WINDOWS) && !defined(DONT_USE_SEH) #define HANDLE_INVALID_MEM_METHOD … #else #define HANDLE_INVALID_MEM_METHOD(self, sourcecode) … #endif int safe_memcpy(void *dest, const void *src, size_t count) { … } int safe_byte_copy(char *dest, const char *src) { … } int safe_memchr(char **out, const void *ptr, int ch, size_t count) { … } int safe_memmove(void *dest, const void *src, size_t count) { … } int safe_copy_from_slice(char *dest, const char *src, Py_ssize_t start, Py_ssize_t step, Py_ssize_t slicelen) { … } int safe_copy_to_slice(char *dest, const char *src, Py_ssize_t start, Py_ssize_t step, Py_ssize_t slicelen) { … } int _safe_PyBytes_Find(Py_ssize_t *out, mmap_object *self, const char *haystack, Py_ssize_t len_haystack, const char *needle, Py_ssize_t len_needle, Py_ssize_t offset) { … } int _safe_PyBytes_ReverseFind(Py_ssize_t *out, mmap_object *self, const char *haystack, Py_ssize_t len_haystack, const char *needle, Py_ssize_t len_needle, Py_ssize_t offset) { … } PyObject * _safe_PyBytes_FromStringAndSize(char *start, size_t num_bytes) { … } static PyObject * mmap_read_byte_method(mmap_object *self, PyObject *Py_UNUSED(ignored)) { … } static PyObject * mmap_read_line_method(mmap_object *self, PyObject *Py_UNUSED(ignored)) { … } static PyObject * mmap_read_method(mmap_object *self, PyObject *args) { … } static PyObject * mmap_gfind(mmap_object *self, PyObject *args, int reverse) { … } static PyObject * mmap_find_method(mmap_object *self, PyObject *args) { … } static PyObject * mmap_rfind_method(mmap_object *self, PyObject *args) { … } static int is_writable(mmap_object *self) { … } static int is_resizeable(mmap_object *self) { … } static PyObject * mmap_write_method(mmap_object *self, PyObject *args) { … } static PyObject * mmap_write_byte_method(mmap_object *self, PyObject *args) { … } static PyObject * mmap_size_method(mmap_object *self, PyObject *Py_UNUSED(ignored)) { … } /* This assumes that you want the entire file mapped, / and when recreating the map will make the new file / have the new size / / Is this really necessary? This could easily be done / from python by just closing and re-opening with the / new size? */ static PyObject * mmap_resize_method(mmap_object *self, PyObject *args) { … } static PyObject * mmap_tell_method(mmap_object *self, PyObject *Py_UNUSED(ignored)) { … } static PyObject * mmap_flush_method(mmap_object *self, PyObject *args) { … } static PyObject * mmap_seek_method(mmap_object *self, PyObject *args) { … } static PyObject * mmap_seekable_method(mmap_object *self, PyObject *Py_UNUSED(ignored)) { … } static PyObject * mmap_move_method(mmap_object *self, PyObject *args) { … } static PyObject * mmap_closed_get(mmap_object *self, void *Py_UNUSED(ignored)) { … } static PyObject * mmap__enter__method(mmap_object *self, PyObject *args) { … } static PyObject * mmap__exit__method(PyObject *self, PyObject *args) { … } static PyObject * mmap__repr__method(PyObject *self) { … } #ifdef MS_WINDOWS static PyObject * mmap__sizeof__method(mmap_object *self, void *Py_UNUSED(ignored)) { size_t res = _PyObject_SIZE(Py_TYPE(self)); if (self->tagname) { res += (wcslen(self->tagname) + 1) * sizeof(self->tagname[0]); } return PyLong_FromSize_t(res); } #endif #if defined(MS_WINDOWS) && defined(Py_DEBUG) static PyObject * mmap_protect_method(mmap_object *self, PyObject *args) { DWORD flNewProtect, flOldProtect; Py_ssize_t start, length; CHECK_VALID(NULL); if (!PyArg_ParseTuple(args, "Inn:protect", &flNewProtect, &start, &length)) { return NULL; } if (!VirtualProtect((void *) (self->data + start), length, flNewProtect, &flOldProtect)) { PyErr_SetFromWindowsErr(GetLastError()); return NULL; } Py_RETURN_NONE; } #endif #ifdef HAVE_MADVISE static PyObject * mmap_madvise_method(mmap_object *self, PyObject *args) { … } #endif // HAVE_MADVISE static struct PyMemberDef mmap_object_members[] = …; static struct PyMethodDef mmap_object_methods[] = …; static PyGetSetDef mmap_object_getset[] = …; /* Functions for treating an mmap'ed file as a buffer */ static int mmap_buffer_getbuf(mmap_object *self, Py_buffer *view, int flags) { … } static void mmap_buffer_releasebuf(mmap_object *self, Py_buffer *view) { … } static Py_ssize_t mmap_length(mmap_object *self) { … } static PyObject * mmap_item(mmap_object *self, Py_ssize_t i) { … } static PyObject * mmap_subscript(mmap_object *self, PyObject *item) { … } static int mmap_ass_item(mmap_object *self, Py_ssize_t i, PyObject *v) { … } static int mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) { … } static PyObject * new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict); PyDoc_STRVAR(mmap_doc, "Windows: mmap(fileno, length[, tagname[, access[, offset]]])\n\ \n\ Maps length bytes from the file specified by the file handle fileno,\n\ and returns a mmap object. If length is larger than the current size\n\ of the file, the file is extended to contain length bytes. If length\n\ is 0, the maximum length of the map is the current size of the file,\n\ except that if the file is empty Windows raises an exception (you cannot\n\ create an empty mapping on Windows).\n\ \n\ Unix: mmap(fileno, length[, flags[, prot[, access[, offset[, trackfd]]]]])\n\ \n\ Maps length bytes from the file specified by the file descriptor fileno,\n\ and returns a mmap object. If length is 0, the maximum length of the map\n\ will be the current size of the file when mmap is called.\n\ flags specifies the nature of the mapping. MAP_PRIVATE creates a\n\ private copy-on-write mapping, so changes to the contents of the mmap\n\ object will be private to this process, and MAP_SHARED creates a mapping\n\ that's shared with all other processes mapping the same areas of the file.\n\ The default value is MAP_SHARED.\n\ \n\ To map anonymous memory, pass -1 as the fileno (both versions)."); static PyType_Slot mmap_object_slots[] = …; static PyType_Spec mmap_object_spec = …; #ifdef UNIX #ifdef HAVE_LARGEFILE_SUPPORT #define _Py_PARSE_OFF_T … #else #define _Py_PARSE_OFF_T … #endif static PyObject * new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) { … } #endif /* UNIX */ #ifdef MS_WINDOWS /* A note on sizes and offsets: while the actual map size must hold in a Py_ssize_t, both the total file size and the start offset can be longer than a Py_ssize_t, so we use long long which is always 64-bit. */ static PyObject * new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) { mmap_object *m_obj; Py_ssize_t map_size; long long offset = 0, size; DWORD off_hi; /* upper 32 bits of offset */ DWORD off_lo; /* lower 32 bits of offset */ DWORD size_hi; /* upper 32 bits of size */ DWORD size_lo; /* lower 32 bits of size */ PyObject *tagname = Py_None; DWORD dwErr = 0; int fileno; HANDLE fh = 0; int access = (access_mode)ACCESS_DEFAULT; DWORD flProtect, dwDesiredAccess; static char *keywords[] = { "fileno", "length", "tagname", "access", "offset", NULL }; if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|OiL", keywords, &fileno, &map_size, &tagname, &access, &offset)) { return NULL; } if (PySys_Audit("mmap.__new__", "iniL", fileno, map_size, access, offset) < 0) { return NULL; } switch((access_mode)access) { case ACCESS_READ: flProtect = PAGE_READONLY; dwDesiredAccess = FILE_MAP_READ; break; case ACCESS_DEFAULT: case ACCESS_WRITE: flProtect = PAGE_READWRITE; dwDesiredAccess = FILE_MAP_WRITE; break; case ACCESS_COPY: flProtect = PAGE_WRITECOPY; dwDesiredAccess = FILE_MAP_COPY; break; default: return PyErr_Format(PyExc_ValueError, "mmap invalid access parameter."); } if (map_size < 0) { PyErr_SetString(PyExc_OverflowError, "memory mapped length must be positive"); return NULL; } if (offset < 0) { PyErr_SetString(PyExc_OverflowError, "memory mapped offset must be positive"); return NULL; } /* assume -1 and 0 both mean invalid filedescriptor to 'anonymously' map memory. XXX: fileno == 0 is a valid fd, but was accepted prior to 2.5. XXX: Should this code be added? if (fileno == 0) PyErr_WarnEx(PyExc_DeprecationWarning, "don't use 0 for anonymous memory", 1); */ if (fileno != -1 && fileno != 0) { /* Ensure that fileno is within the CRT's valid range */ fh = _Py_get_osfhandle(fileno); if (fh == INVALID_HANDLE_VALUE) return NULL; /* Win9x appears to need us seeked to zero */ lseek(fileno, 0, SEEK_SET); } m_obj = (mmap_object *)type->tp_alloc(type, 0); if (m_obj == NULL) return NULL; /* Set every field to an invalid marker, so we can safely destruct the object in the face of failure */ m_obj->data = NULL; m_obj->file_handle = INVALID_HANDLE_VALUE; m_obj->map_handle = NULL; m_obj->tagname = NULL; m_obj->offset = offset; if (fh) { /* It is necessary to duplicate the handle, so the Python code can close it on us */ if (!DuplicateHandle( GetCurrentProcess(), /* source process handle */ fh, /* handle to be duplicated */ GetCurrentProcess(), /* target proc handle */ (LPHANDLE)&m_obj->file_handle, /* result */ 0, /* access - ignored due to options value */ FALSE, /* inherited by child processes? */ DUPLICATE_SAME_ACCESS)) { /* options */ dwErr = GetLastError(); Py_DECREF(m_obj); PyErr_SetFromWindowsErr(dwErr); return NULL; } if (!map_size) { DWORD low,high; low = GetFileSize(fh, &high); /* low might just happen to have the value INVALID_FILE_SIZE; so we need to check the last error also. */ if (low == INVALID_FILE_SIZE && (dwErr = GetLastError()) != NO_ERROR) { Py_DECREF(m_obj); return PyErr_SetFromWindowsErr(dwErr); } size = (((long long) high) << 32) + low; if (size == 0) { PyErr_SetString(PyExc_ValueError, "cannot mmap an empty file"); Py_DECREF(m_obj); return NULL; } if (offset >= size) { PyErr_SetString(PyExc_ValueError, "mmap offset is greater than file size"); Py_DECREF(m_obj); return NULL; } if (size - offset > PY_SSIZE_T_MAX) { PyErr_SetString(PyExc_ValueError, "mmap length is too large"); Py_DECREF(m_obj); return NULL; } m_obj->size = (Py_ssize_t) (size - offset); } else { m_obj->size = map_size; size = offset + map_size; } } else { m_obj->size = map_size; size = offset + map_size; } /* set the initial position */ m_obj->pos = (size_t) 0; m_obj->weakreflist = NULL; m_obj->exports = 0; /* set the tag name */ if (!Py_IsNone(tagname)) { if (!PyUnicode_Check(tagname)) { Py_DECREF(m_obj); return PyErr_Format(PyExc_TypeError, "expected str or None for " "'tagname', not %.200s", Py_TYPE(tagname)->tp_name); } m_obj->tagname = PyUnicode_AsWideCharString(tagname, NULL); if (m_obj->tagname == NULL) { Py_DECREF(m_obj); return NULL; } } m_obj->access = (access_mode)access; size_hi = (DWORD)(size >> 32); size_lo = (DWORD)(size & 0xFFFFFFFF); off_hi = (DWORD)(offset >> 32); off_lo = (DWORD)(offset & 0xFFFFFFFF); /* For files, it would be sufficient to pass 0 as size. For anonymous maps, we have to pass the size explicitly. */ m_obj->map_handle = CreateFileMappingW(m_obj->file_handle, NULL, flProtect, size_hi, size_lo, m_obj->tagname); if (m_obj->map_handle != NULL) { m_obj->data = (char *) MapViewOfFile(m_obj->map_handle, dwDesiredAccess, off_hi, off_lo, m_obj->size); if (m_obj->data != NULL) return (PyObject *)m_obj; else { dwErr = GetLastError(); CloseHandle(m_obj->map_handle); m_obj->map_handle = NULL; } } else dwErr = GetLastError(); Py_DECREF(m_obj); PyErr_SetFromWindowsErr(dwErr); return NULL; } #endif /* MS_WINDOWS */ static int mmap_exec(PyObject *module) { … } static PyModuleDef_Slot mmap_slots[] = …; static struct PyModuleDef mmapmodule = …; PyMODINIT_FUNC PyInit_mmap(void) { … }
Codebase Browser
cpython