/* C Extension module to test all aspects of PEP-3118. Written by Stefan Krah. */ #include "Python.h" /* struct module */ static PyObject *structmodule = …; static PyObject *Struct = …; static PyObject *calcsize = …; /* cache simple format string */ static const char *simple_fmt = …; static PyObject *simple_format = …; #define SIMPLE_FORMAT(fmt) … #define FIX_FORMAT(fmt) … /**************************************************************************/ /* NDArray Object */ /**************************************************************************/ static PyTypeObject NDArray_Type; #define NDArray_Check(v) … #define CHECK_LIST_OR_TUPLE(v) … #define PyMem_XFree(v) … /* Maximum number of dimensions. */ #define ND_MAX_NDIM … /* Check for the presence of suboffsets in the first dimension. */ #define HAVE_PTR(suboffsets) … /* Adjust ptr if suboffsets are present. */ #define ADJUST_PTR(ptr, suboffsets) … /* Default: NumPy style (strides), read-only, no var-export, C-style layout */ #define ND_DEFAULT … /* User configurable flags for the ndarray */ #define ND_VAREXPORT … /* User configurable flags for each base buffer */ #define ND_WRITABLE … #define ND_FORTRAN … #define ND_SCALAR … #define ND_PIL … #define ND_REDIRECT … #define ND_GETBUF_FAIL … #define ND_GETBUF_UNDEFINED … /* Internal flags for the base buffer */ #define ND_C … #define ND_OWN_ARRAYS … /* ndarray properties */ #define ND_IS_CONSUMER(nd) … /* ndbuf->flags properties */ #define ND_C_CONTIGUOUS(flags) … #define ND_FORTRAN_CONTIGUOUS(flags) … #define ND_ANY_CONTIGUOUS(flags) … /* getbuffer() requests */ #define REQ_INDIRECT(flags) … #define REQ_C_CONTIGUOUS(flags) … #define REQ_F_CONTIGUOUS(flags) … #define REQ_ANY_CONTIGUOUS(flags) … #define REQ_STRIDES(flags) … #define REQ_SHAPE(flags) … #define REQ_WRITABLE(flags) … #define REQ_FORMAT(flags) … /* Single node of a list of base buffers. The list is needed to implement changes in memory layout while exported buffers are active. */ static PyTypeObject NDArray_Type; struct ndbuf; ndbuf_t; NDArrayObject; static ndbuf_t * ndbuf_new(Py_ssize_t nitems, Py_ssize_t itemsize, Py_ssize_t offset, int flags) { … } static void ndbuf_free(ndbuf_t *ndbuf) { … } static void ndbuf_push(NDArrayObject *nd, ndbuf_t *elt) { … } static void ndbuf_delete(NDArrayObject *nd, ndbuf_t *elt) { … } static void ndbuf_pop(NDArrayObject *nd) { … } static PyObject * ndarray_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { … } static void ndarray_dealloc(NDArrayObject *self) { … } static int ndarray_init_staticbuf(PyObject *exporter, NDArrayObject *nd, int flags) { … } static void init_flags(ndbuf_t *ndbuf) { … } /****************************************************************************/ /* Buffer/List conversions */ /****************************************************************************/ static Py_ssize_t *strides_from_shape(const ndbuf_t *, int flags); /* Get number of members in a struct: see issue #12740 */ PyPartialStructObject; static Py_ssize_t get_nmemb(PyObject *s) { … } /* Pack all items into the buffer of 'obj'. The 'format' parameter must be in struct module syntax. For standard C types, a single item is an integer. For compound types, a single item is a tuple of integers. */ static int pack_from_list(PyObject *obj, PyObject *items, PyObject *format, Py_ssize_t itemsize) { … } /* Pack single element */ static int pack_single(char *ptr, PyObject *item, const char *fmt, Py_ssize_t itemsize) { … } static void copy_rec(const Py_ssize_t *shape, Py_ssize_t ndim, Py_ssize_t itemsize, char *dptr, const Py_ssize_t *dstrides, const Py_ssize_t *dsuboffsets, char *sptr, const Py_ssize_t *sstrides, const Py_ssize_t *ssuboffsets, char *mem) { … } static int cmp_structure(Py_buffer *dest, Py_buffer *src) { … } /* Copy src to dest. Both buffers must have the same format, itemsize, ndim and shape. Copying is atomic, the function never fails with a partial copy. */ static int copy_buffer(Py_buffer *dest, Py_buffer *src) { … } /* Unpack single element */ static PyObject * unpack_single(char *ptr, const char *fmt, Py_ssize_t itemsize) { … } /* Unpack a multi-dimensional matrix into a nested list. Return a scalar for ndim = 0. */ static PyObject * unpack_rec(PyObject *unpack_from, char *ptr, PyObject *mview, char *item, const Py_ssize_t *shape, const Py_ssize_t *strides, const Py_ssize_t *suboffsets, Py_ssize_t ndim, Py_ssize_t itemsize) { … } static PyObject * ndarray_as_list(NDArrayObject *nd) { … } /****************************************************************************/ /* Initialize ndbuf */ /****************************************************************************/ /* State of a new ndbuf during initialization. 'OK' means that initialization is complete. 'PTR' means that a pointer has been initialized, but the state of the memory is still undefined and ndbuf->offset is disregarded. +-----------------+-----------+-------------+----------------+ | | ndbuf_new | init_simple | init_structure | +-----------------+-----------+-------------+----------------+ | next | OK (NULL) | OK | OK | +-----------------+-----------+-------------+----------------+ | prev | OK (NULL) | OK | OK | +-----------------+-----------+-------------+----------------+ | len | OK | OK | OK | +-----------------+-----------+-------------+----------------+ | offset | OK | OK | OK | +-----------------+-----------+-------------+----------------+ | data | PTR | OK | OK | +-----------------+-----------+-------------+----------------+ | flags | user | user | OK | +-----------------+-----------+-------------+----------------+ | exports | OK (0) | OK | OK | +-----------------+-----------+-------------+----------------+ | base.obj | OK (NULL) | OK | OK | +-----------------+-----------+-------------+----------------+ | base.buf | PTR | PTR | OK | +-----------------+-----------+-------------+----------------+ | base.len | len(data) | len(data) | OK | +-----------------+-----------+-------------+----------------+ | base.itemsize | 1 | OK | OK | +-----------------+-----------+-------------+----------------+ | base.readonly | 0 | OK | OK | +-----------------+-----------+-------------+----------------+ | base.format | NULL | OK | OK | +-----------------+-----------+-------------+----------------+ | base.ndim | 1 | 1 | OK | +-----------------+-----------+-------------+----------------+ | base.shape | NULL | NULL | OK | +-----------------+-----------+-------------+----------------+ | base.strides | NULL | NULL | OK | +-----------------+-----------+-------------+----------------+ | base.suboffsets | NULL | NULL | OK | +-----------------+-----------+-------------+----------------+ | base.internal | OK | OK | OK | +-----------------+-----------+-------------+----------------+ */ static Py_ssize_t get_itemsize(PyObject *format) { … } static char * get_format(PyObject *format) { … } static int init_simple(ndbuf_t *ndbuf, PyObject *items, PyObject *format, Py_ssize_t itemsize) { … } static Py_ssize_t * seq_as_ssize_array(PyObject *seq, Py_ssize_t len, int is_shape) { … } static Py_ssize_t * strides_from_shape(const ndbuf_t *ndbuf, int flags) { … } /* Bounds check: len := complete length of allocated memory offset := start of the array A single array element is indexed by: i = indices[0] * strides[0] + indices[1] * strides[1] + ... imin is reached when all indices[n] combined with positive strides are 0 and all indices combined with negative strides are shape[n]-1, which is the maximum index for the nth dimension. imax is reached when all indices[n] combined with negative strides are 0 and all indices combined with positive strides are shape[n]-1. */ static int verify_structure(Py_ssize_t len, Py_ssize_t itemsize, Py_ssize_t offset, const Py_ssize_t *shape, const Py_ssize_t *strides, Py_ssize_t ndim) { … } /* Convert a NumPy-style array to an array using suboffsets to stride in the first dimension. Requirements: ndim > 0. Contiguous example ================== Input: ------ shape = {2, 2, 3}; strides = {6, 3, 1}; suboffsets = NULL; data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; buf = &data[0] Output: ------- shape = {2, 2, 3}; strides = {sizeof(char *), 3, 1}; suboffsets = {0, -1, -1}; data = {p1, p2, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; | | ^ ^ `---'---' | | | `---------------------' buf = &data[0] So, in the example the input resembles the three-dimensional array char v[2][2][3], while the output resembles an array of two pointers to two-dimensional arrays: char (*v[2])[2][3]. Non-contiguous example: ======================= Input (with offset and negative strides): ----------------------------------------- shape = {2, 2, 3}; strides = {-6, 3, -1}; offset = 8 suboffsets = NULL; data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; Output: ------- shape = {2, 2, 3}; strides = {-sizeof(char *), 3, -1}; suboffsets = {2, -1, -1}; newdata = {p1, p2, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; | | ^ ^ ^ ^ `---'---' | | `- p2+suboffsets[0] | `-----------|--- p1+suboffsets[0] `---------------------' buf = &newdata[1] # striding backwards over the pointers. suboffsets[0] is the same as the offset that one would specify if the two {2, 3} subarrays were created directly, hence the name. */ static int init_suboffsets(ndbuf_t *ndbuf) { … } static void init_len(Py_buffer *base) { … } static int init_structure(ndbuf_t *ndbuf, PyObject *shape, PyObject *strides, Py_ssize_t ndim) { … } static ndbuf_t * init_ndbuf(PyObject *items, PyObject *shape, PyObject *strides, Py_ssize_t offset, PyObject *format, int flags) { … } /* initialize and push a new base onto the linked list */ static int ndarray_push_base(NDArrayObject *nd, PyObject *items, PyObject *shape, PyObject *strides, Py_ssize_t offset, PyObject *format, int flags) { … } #define PyBUF_UNUSED … static int ndarray_init(PyObject *self, PyObject *args, PyObject *kwds) { … } /* Push an additional base onto the linked list. */ static PyObject * ndarray_push(PyObject *self, PyObject *args, PyObject *kwds) { … } /* Pop a base from the linked list (if possible). */ static PyObject * ndarray_pop(PyObject *self, PyObject *dummy) { … } /**************************************************************************/ /* getbuffer */ /**************************************************************************/ static int ndarray_getbuf(NDArrayObject *self, Py_buffer *view, int flags) { … } static void ndarray_releasebuf(NDArrayObject *self, Py_buffer *view) { … } static PyBufferProcs ndarray_as_buffer = …; /**************************************************************************/ /* indexing/slicing */ /**************************************************************************/ static char * ptr_from_index(Py_buffer *base, Py_ssize_t index) { … } static PyObject * ndarray_item(NDArrayObject *self, Py_ssize_t index) { … } /* For each dimension, we get valid (start, stop, step, slicelength) quadruples from PySlice_GetIndicesEx(). Slicing NumPy arrays ==================== A pointer to an element in a NumPy array is defined by: ptr = (char *)buf + indices[0] * strides[0] + ... + indices[ndim-1] * strides[ndim-1] Adjust buf: ----------- Adding start[n] for each dimension effectively adds the constant: c = start[0] * strides[0] + ... + start[ndim-1] * strides[ndim-1] Therefore init_slice() adds all start[n] directly to buf. Adjust shape: ------------- Obviously shape[n] = slicelength[n] Adjust strides: --------------- In the original array, the next element in a dimension is reached by adding strides[n] to the pointer. In the sliced array, elements may be skipped, so the next element is reached by adding: strides[n] * step[n] Slicing PIL arrays ================== Layout: ------- In the first (zeroth) dimension, PIL arrays have an array of pointers to sub-arrays of ndim-1. Striding in the first dimension is done by getting the index of the nth pointer, dereference it and then add a suboffset to it. The arrays pointed to can best be seen a regular NumPy arrays. Adjust buf: ----------- In the original array, buf points to a location (usually the start) in the array of pointers. For the sliced array, start[0] can be added to buf in the same manner as for NumPy arrays. Adjust suboffsets: ------------------ Due to the dereferencing step in the addressing scheme, it is not possible to adjust buf for higher dimensions. Recall that the sub-arrays pointed to are regular NumPy arrays, so for each of those arrays adding start[n] effectively adds the constant: c = start[1] * strides[1] + ... + start[ndim-1] * strides[ndim-1] This constant is added to suboffsets[0]. suboffsets[0] in turn is added to each pointer right after dereferencing. Adjust shape and strides: ------------------------- Shape and strides are not influenced by the dereferencing step, so they are adjusted in the same manner as for NumPy arrays. Multiple levels of suboffsets ============================= For a construct like an array of pointers to array of pointers to sub-arrays of ndim-2: suboffsets[0] = start[1] * strides[1] suboffsets[1] = start[2] * strides[2] + ... */ static int init_slice(Py_buffer *base, PyObject *key, int dim) { … } static int copy_structure(Py_buffer *base) { … } static PyObject * ndarray_subscript(NDArrayObject *self, PyObject *key) { … } static int ndarray_ass_subscript(NDArrayObject *self, PyObject *key, PyObject *value) { … } static PyObject * slice_indices(PyObject *self, PyObject *args) { … } static PyMappingMethods ndarray_as_mapping = …; static PySequenceMethods ndarray_as_sequence = …; /**************************************************************************/ /* getters */ /**************************************************************************/ static PyObject * ssize_array_as_tuple(Py_ssize_t *array, Py_ssize_t len) { … } static PyObject * ndarray_get_flags(NDArrayObject *self, void *closure) { … } static PyObject * ndarray_get_offset(NDArrayObject *self, void *closure) { … } static PyObject * ndarray_get_obj(NDArrayObject *self, void *closure) { … } static PyObject * ndarray_get_nbytes(NDArrayObject *self, void *closure) { … } static PyObject * ndarray_get_readonly(NDArrayObject *self, void *closure) { … } static PyObject * ndarray_get_itemsize(NDArrayObject *self, void *closure) { … } static PyObject * ndarray_get_format(NDArrayObject *self, void *closure) { … } static PyObject * ndarray_get_ndim(NDArrayObject *self, void *closure) { … } static PyObject * ndarray_get_shape(NDArrayObject *self, void *closure) { … } static PyObject * ndarray_get_strides(NDArrayObject *self, void *closure) { … } static PyObject * ndarray_get_suboffsets(NDArrayObject *self, void *closure) { … } static PyObject * ndarray_c_contig(PyObject *self, PyObject *dummy) { … } static PyObject * ndarray_fortran_contig(PyObject *self, PyObject *dummy) { … } static PyObject * ndarray_contig(PyObject *self, PyObject *dummy) { … } static PyGetSetDef ndarray_getset [] = …; static PyObject * ndarray_tolist(PyObject *self, PyObject *dummy) { … } static PyObject * ndarray_tobytes(PyObject *self, PyObject *dummy) { … } /* add redundant (negative) suboffsets for testing */ static PyObject * ndarray_add_suboffsets(PyObject *self, PyObject *dummy) { … } /* Test PyMemoryView_FromBuffer(): return a memoryview from a static buffer. Obviously this is fragile and only one such view may be active at any time. Never use anything like this in real code! */ static char *infobuf = …; static PyObject * ndarray_memoryview_from_buffer(PyObject *self, PyObject *dummy) { … } /* Get a single item from bufobj at the location specified by seq. seq is a list or tuple of indices. The purpose of this function is to check other functions against PyBuffer_GetPointer(). */ static PyObject * get_pointer(PyObject *self, PyObject *args) { … } static PyObject * get_sizeof_void_p(PyObject *self, PyObject *Py_UNUSED(ignored)) { … } static char get_ascii_order(PyObject *order) { … } /* Get a contiguous memoryview. */ static PyObject * get_contiguous(PyObject *self, PyObject *args) { … } /* PyBuffer_ToContiguous() */ static PyObject * py_buffer_to_contiguous(PyObject *self, PyObject *args) { … } static int fmtcmp(const char *fmt1, const char *fmt2) { … } static int arraycmp(const Py_ssize_t *a1, const Py_ssize_t *a2, const Py_ssize_t *shape, Py_ssize_t ndim) { … } /* Compare two contiguous buffers for physical equality. */ static PyObject * cmp_contig(PyObject *self, PyObject *args) { … } static PyObject * is_contiguous(PyObject *self, PyObject *args) { … } static Py_hash_t ndarray_hash(PyObject *self) { … } static PyMethodDef ndarray_methods [] = …; static PyTypeObject NDArray_Type = …; /**************************************************************************/ /* StaticArray Object */ /**************************************************************************/ static PyTypeObject StaticArray_Type; StaticArrayObject; static char static_mem[12] = …; static Py_ssize_t static_shape[1] = …; static Py_ssize_t static_strides[1] = …; static Py_buffer static_buffer = …; static PyObject * staticarray_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { … } static int staticarray_init(PyObject *self, PyObject *args, PyObject *kwds) { … } static void staticarray_dealloc(StaticArrayObject *self) { … } /* Return a buffer for a PyBUF_FULL_RO request. Flags are not checked, which makes this object a non-compliant exporter! */ static int staticarray_getbuf(StaticArrayObject *self, Py_buffer *view, int flags) { … } static PyBufferProcs staticarray_as_buffer = …; static PyTypeObject StaticArray_Type = …; static struct PyMethodDef _testbuffer_functions[] = …; static struct PyModuleDef _testbuffermodule = …; static int _testbuffer_exec(PyObject *mod) { … } PyMODINIT_FUNC PyInit__testbuffer(void) { … }
Codebase Browser
cpython