From 810364c78185b8cb16a3579162a1e701cf0bf410 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Sat, 1 Mar 2025 13:13:18 +0000 Subject: [PATCH] Remove deprecated blosc code --- docs/release.rst | 22 +++++- numcodecs/__init__.py | 2 +- numcodecs/blosc.pyx | 131 ---------------------------------- numcodecs/tests/common.py | 68 ------------------ numcodecs/tests/test_blosc.py | 14 ---- pyproject.toml | 7 +- 6 files changed, 24 insertions(+), 220 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index c5ba3919..286e534b 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -17,6 +17,26 @@ Release notes Unreleased ---------- +Removals +~~~~~~~~ + +The following ``blosc`` funcitons are removed, with no replacement. +This is because they were not intended to be public API. + +- ``numcodecs.blosc.init`` +- ``numcodecs.blosc.destroy`` +- ``numcodecs.blosc.compname_to_compcode`` +- ``numcodecs.blosc.cbuffer_sizes`` +- ``numcodecs.blosc.cbuffer_metainfo`` + +In addition, ``numcodecs.blosc.decompress_partial`` is removed as +has always been experimental and there is no equivalent in the official +blsoc Python package. +By :user:`David Stansby `, :issue:`712` + +0.15.1 +------ + Improvements ~~~~~~~~~~~~ * Raise a custom `UnknownCodecError` when trying to retrieve an unavailable codec. @@ -52,7 +72,7 @@ This is because they are not intended to be public API. In addition, ``numcodecs.blosc.decompress_partial`` is deprecated as has always been experimental and there is no equivalent in the official blsoc Python package. -By :user:`David Stansby `, :issue`619` +By :user:`David Stansby `, :issue:`619` Fixes ~~~~~ diff --git a/numcodecs/__init__.py b/numcodecs/__init__.py index b4b983ce..e3626b1b 100644 --- a/numcodecs/__init__.py +++ b/numcodecs/__init__.py @@ -51,7 +51,7 @@ ncores = 1 blosc._init() blosc.set_nthreads(min(8, ncores)) -atexit.register(blosc.destroy) +atexit.register(blosc._destroy) from numcodecs import zstd as zstd from numcodecs.zstd import Zstd diff --git a/numcodecs/blosc.pyx b/numcodecs/blosc.pyx index 3caa3607..13480f63 100644 --- a/numcodecs/blosc.pyx +++ b/numcodecs/blosc.pyx @@ -6,7 +6,6 @@ import threading import multiprocessing import os -from deprecated import deprecated from cpython.buffer cimport PyBUF_ANY_CONTIGUOUS, PyBUF_WRITEABLE @@ -45,7 +44,6 @@ cdef extern from "blosc.h": void* src, void* dest, size_t destsize) nogil int blosc_decompress(void *src, void *dest, size_t destsize) nogil int blosc_getitem(void* src, int start, int nitems, void* dest) - int blosc_compname_to_compcode(const char* compname) int blosc_compress_ctx(int clevel, int doshuffle, size_t typesize, size_t nbytes, const void* src, void* dest, size_t destsize, const char* compressor, size_t blocksize, @@ -100,28 +98,12 @@ def _init(): """Initialize the Blosc library environment.""" blosc_init() -init = deprecated(_init) - def _destroy(): """Destroy the Blosc library environment.""" blosc_destroy() -destroy = deprecated(_destroy) - - -def _compname_to_compcode(cname): - """Return the compressor code associated with the compressor name. If the compressor - name is not recognized, or there is not support for it in this build, -1 is returned - instead.""" - if isinstance(cname, str): - cname = cname.encode('ascii') - return blosc_compname_to_compcode(cname) - -compname_to_compcode = deprecated(_compname_to_compcode) - - def list_compressors(): """Get a list of compressors supported in the current build.""" s = blosc_list_compressors() @@ -141,35 +123,6 @@ def set_nthreads(int nthreads): return blosc_set_nthreads(nthreads) -def _cbuffer_sizes(source): - """Return information about a compressed buffer, namely the number of uncompressed - bytes (`nbytes`) and compressed (`cbytes`). It also returns the `blocksize` (which - is used internally for doing the compression by blocks). - - Returns - ------- - nbytes : int - cbytes : int - blocksize : int - - """ - cdef: - Buffer buffer - size_t nbytes, cbytes, blocksize - - # obtain buffer - buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS) - - # determine buffer size - blosc_cbuffer_sizes(buffer.ptr, &nbytes, &cbytes, &blocksize) - - # release buffers - buffer.release() - - return nbytes, cbytes, blocksize - -cbuffer_sizes = deprecated(_cbuffer_sizes) - def cbuffer_complib(source): """Return the name of the compression library used to compress `source`.""" cdef: @@ -226,13 +179,10 @@ def _cbuffer_metainfo(source): return typesize, shuffle, memcpyed -cbuffer_metainfo = deprecated(_cbuffer_metainfo) - def _err_bad_cname(cname): raise ValueError('bad compressor or compressor not supported: %r; expected one of ' '%s' % (cname, list_compressors())) -err_bad_cname = deprecated(_err_bad_cname) def compress(source, char* cname, int clevel, int shuffle=SHUFFLE, int blocksize=AUTOBLOCKS): @@ -415,82 +365,6 @@ def decompress(source, dest=None): return dest - -def _decompress_partial(source, start, nitems, dest=None): - """**Experimental** - Decompress data of only a part of a buffer. - - Parameters - ---------- - source : bytes-like - Compressed data, including blosc header. Can be any object supporting the buffer - protocol. - start: int, - Offset in item where we want to start decoding - nitems: int - Number of items we want to decode - dest : array-like, optional - Object to decompress into. - - - Returns - ------- - dest : bytes - Object containing decompressed data. - - """ - cdef: - int ret - int encoding_size - int nitems_bytes - int start_bytes - char *source_ptr - char *dest_ptr - Buffer source_buffer - Buffer dest_buffer = None - - # setup source buffer - source_buffer = Buffer(source, PyBUF_ANY_CONTIGUOUS) - source_ptr = source_buffer.ptr - - # get encoding size from source buffer header - encoding_size = source[3] - - # convert variables to handle type and encoding sizes - nitems_bytes = nitems * encoding_size - start_bytes = (start * encoding_size) - - # setup destination buffer - if dest is None: - dest = PyBytes_FromStringAndSize(NULL, nitems_bytes) - dest_ptr = PyBytes_AS_STRING(dest) - dest_nbytes = nitems_bytes - else: - arr = ensure_contiguous_ndarray(dest) - dest_buffer = Buffer(arr, PyBUF_ANY_CONTIGUOUS | PyBUF_WRITEABLE) - dest_ptr = dest_buffer.ptr - dest_nbytes = dest_buffer.nbytes - - # try decompression - try: - if dest_nbytes < nitems_bytes: - raise ValueError('destination buffer too small; expected at least %s, ' - 'got %s' % (nitems_bytes, dest_nbytes)) - ret = blosc_getitem(source_ptr, start, nitems, dest_ptr) - - finally: - source_buffer.release() - if dest_buffer is not None: - dest_buffer.release() - - # ret refers to the number of bytes returned from blosc_getitem. - if ret <= 0: - raise RuntimeError('error during blosc partial decompression: %d', ret) - - return dest - -decompress_partial = deprecated(_decompress_partial) - # set the value of this variable to True or False to override the # default adaptive behaviour use_threads = None @@ -584,11 +458,6 @@ class Blosc(Codec): buf = ensure_contiguous_ndarray(buf, self.max_buffer_size) return decompress(buf, out) - def decode_partial(self, buf, int start, int nitems, out=None): - '''**Experimental**''' - buf = ensure_contiguous_ndarray(buf, self.max_buffer_size) - return _decompress_partial(buf, start, nitems, dest=out) - def __repr__(self): r = '%s(cname=%r, clevel=%r, shuffle=%s, blocksize=%s)' % \ (type(self).__name__, diff --git a/numcodecs/tests/common.py b/numcodecs/tests/common.py index bb7c4780..12cccb20 100644 --- a/numcodecs/tests/common.py +++ b/numcodecs/tests/common.py @@ -115,74 +115,6 @@ def check_encode_decode(arr, codec, precision=None): compare_arrays(arr, out, precision=precision) -def check_encode_decode_partial(arr, codec, precision=None): - # N.B., watch out here with blosc compressor, if the itemsize of - # the source buffer is different then the results of encoding - # (i.e., compression) may be different. Hence we *do not* require that - # the results of encoding be identical for all possible inputs, rather - # we just require that the results of the encode/decode round-trip can - # be compared to the original array. - - itemsize = arr.itemsize - start, nitems = 5, 10 - compare_arr = arr[start : start + nitems] - # test encoding of numpy array - enc = codec.encode(arr) - dec = codec.decode_partial(enc, start, nitems) - compare_arrays(compare_arr, dec, precision=precision) - - # out = np.empty_like(compare_arr) - out = np.empty_like(compare_arr) - print(len(out)) - - # test partial decode of encoded bytes - buf = arr.tobytes(order='A') - enc = codec.encode(buf) - dec = codec.decode_partial(enc, start * itemsize, nitems * itemsize, out=out) - compare_arrays(compare_arr, dec, precision=precision) - - # test partial decode of encoded bytearray - buf = bytearray(arr.tobytes(order='A')) - enc = codec.encode(buf) - dec = codec.decode_partial(enc, start * itemsize, nitems * itemsize, out=out) - compare_arrays(compare_arr, dec, precision=precision) - - # test partial decode of encoded array.array - buf = array.array('b', arr.tobytes(order='A')) - enc = codec.encode(buf) - dec = codec.decode_partial(enc, start * itemsize, nitems * itemsize, out=out) - compare_arrays(compare_arr, dec, precision=precision) - - # # decoding should support any object exporting the buffer protocol, - - # # setup - enc_bytes = ensure_bytes(enc) - - # test decoding of raw bytes into numpy array - dec = codec.decode_partial(enc_bytes, start * itemsize, nitems * itemsize, out=out) - compare_arrays(compare_arr, dec, precision=precision) - - # test partial decoding of bytearray - dec = codec.decode_partial(bytearray(enc_bytes), start * itemsize, nitems * itemsize, out=out) - compare_arrays(compare_arr, dec, precision=precision) - - # test partial decoding of array.array - buf = array.array('b', enc_bytes) - dec = codec.decode_partial(buf, start * itemsize, nitems * itemsize, out=out) - compare_arrays(compare_arr, dec, precision=precision) - - # test decoding of numpy array into numpy array - buf = np.frombuffer(enc_bytes, dtype='u1') - dec = codec.decode_partial(buf, start * itemsize, nitems * itemsize, out=out) - compare_arrays(compare_arr, dec, precision=precision) - - # test decoding directly into bytearray - out = bytearray(compare_arr.nbytes) - codec.decode_partial(enc_bytes, start * itemsize, nitems * itemsize, out=out) - # noinspection PyTypeChecker - compare_arrays(compare_arr, out, precision=precision) - - def assert_array_items_equal(res, arr): assert isinstance(res, np.ndarray) res = res.reshape(-1, order='A') diff --git a/numcodecs/tests/test_blosc.py b/numcodecs/tests/test_blosc.py index 0bc14010..2cbe9455 100644 --- a/numcodecs/tests/test_blosc.py +++ b/numcodecs/tests/test_blosc.py @@ -15,7 +15,6 @@ check_backwards_compatibility, check_config, check_encode_decode, - check_encode_decode_partial, check_err_decode_object_buffer, check_err_encode_object_buffer, check_max_buffer_size, @@ -75,19 +74,6 @@ def test_encode_decode(array, codec): check_encode_decode(array, codec) -@pytest.mark.parametrize('codec', codecs) -@pytest.mark.parametrize( - 'array', - [ - pytest.param(x) if len(x.shape) == 1 else pytest.param(x, marks=[pytest.mark.xfail]) - for x in arrays - ], -) -def test_partial_decode(codec, array): - _skip_null(codec) - check_encode_decode_partial(array, codec) - - def test_config(): codec = Blosc(cname='zstd', clevel=3, shuffle=1) check_config(codec) diff --git a/pyproject.toml b/pyproject.toml index 29ef0bf0..fae1f9b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,11 +13,8 @@ name = "numcodecs" description = """ A Python package providing buffer compression and transformation codecs \ for use in data storage and communication applications.""" -readme = "README.rst" -dependencies = [ - "numpy>=1.24", - "deprecated" -] +readme = "README.rst" +dependencies = ["numpy>=1.24"] requires-python = ">=3.11" dynamic = [ "version",