From e28a6404a9ec73bb06120ed71dbdc4cd7adc0558 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 18 Oct 2023 10:04:27 -0400 Subject: [PATCH 1/9] MNT: Enable cp312 wheels --- .ci/build_wheels.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/build_wheels.sh b/.ci/build_wheels.sh index 5f7a0b8c..0c3dde20 100755 --- a/.ci/build_wheels.sh +++ b/.ci/build_wheels.sh @@ -24,7 +24,7 @@ export CIBW_TEST_REQUIRES="cython pytest numpy nibabel coverage cython-coverage # compiling numpy takes too long, and causes GHA jobs to time out). # # Disable py312 builds until numpy is available -export CIBW_SKIP="pp* *musllinux* *312*" +export CIBW_SKIP="pp* *musllinux*" # Skip i686/aarch64 tests - I have experienced hangs on these # platforms, which I traced to a trivial numpy operation - From 78acca6af975985be481fcdcd4205fd165d86cd8 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 18 Oct 2023 10:06:00 -0400 Subject: [PATCH 2/9] MNT: Disable deprecated numpy API --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e43112d1..864d6943 100644 --- a/setup.py +++ b/setup.py @@ -125,7 +125,9 @@ def run(self): extra_srcs = [] extra_compile_args = [] compiler_directives = {'language_level' : 2} -define_macros = [] +define_macros = [ + ('NPY_NO_DEPRECATED_API', 'NPY_1_7_API_VERSION'), +] if ZLIB_HOME is not None: include_dirs.append(ZLIB_HOME) From d1bc6b438823384c89ceab13f28864db0ecd02f1 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 18 Oct 2023 10:17:18 -0400 Subject: [PATCH 3/9] CI: Test on Python 3.12 --- .github/workflows/main.yaml | 2 +- .github/workflows/pull_request.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index e1313d63..b82bc063 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -23,7 +23,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] extra-args: ["", "--concat"] env: diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 8d2a6d14..6e030080 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -14,7 +14,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] extra-args: ["", "--concat"] env: From ba38176f8a74b02fd648d5a0cbb685aa868bab86 Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Mon, 23 Oct 2023 14:53:56 +0100 Subject: [PATCH 4/9] TEST: Show local var values on test failure --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 457bf1e2..0ae33384 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ version = {attr = "indexed_gzip.__version__"} [tool.pytest.ini_options] testpaths = ["indexed_gzip/tests"] -addopts = "-v --cov=indexed_gzip" +addopts = "-v --cov=indexed_gzip --showlocals" markers = [ "zran_test: Test the zran.c library", "indexed_gzip_test: Test the indexed_gzip library", From 48194690a1747c64575c8863b1024198dc3b5e9a Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Mon, 23 Oct 2023 14:54:21 +0100 Subject: [PATCH 5/9] CI: Skip win/py312 tests to work around unresolved failures (will revisit in the future). Attempt to enable musllinux and aarch64 builds --- .ci/build_wheels.sh | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.ci/build_wheels.sh b/.ci/build_wheels.sh index 0c3dde20..2ce9a7ee 100755 --- a/.ci/build_wheels.sh +++ b/.ci/build_wheels.sh @@ -19,19 +19,16 @@ export CIBW_TEST_REQUIRES="cython pytest numpy nibabel coverage cython-coverage # Disable pypy builds (reasons for doing this have been lost to # history [GHA logs of failing builds deleted]). -# -# Disable musllinux builds until numpy binaries are available (as -# compiling numpy takes too long, and causes GHA jobs to time out). -# -# Disable py312 builds until numpy is available -export CIBW_SKIP="pp* *musllinux*" +export CIBW_SKIP="pp*" -# Skip i686/aarch64 tests - I have experienced hangs on these +# Skip i686 tests - I have experienced hangs on these # platforms, which I traced to a trivial numpy operation - -# "numpy.linalg.det(numpy.eye(3))". This occurs when numpy has -# to be compiled from source during the build, so can be -# re-visited if/when numpy is avaialble on all platforms. -export CIBW_TEST_SKIP="*i686* *aarch64*" +# "numpy.linalg.det(numpy.eye(3))". This occurs when numpy +# has to be compiled from source during the build, so can +# be re-visited if/when numpy is avaialble on all platforms. +# +# Skip py312 tests on Windows due to unresolved test failures. +export CIBW_TEST_SKIP="*i686* cp312-win*" # Pytest makes it *very* awkward to run tests # from an installed package, and still find/ From 47aa414543ad8e2ed0af81710641766c2c41975a Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Mon, 23 Oct 2023 15:05:16 +0100 Subject: [PATCH 6/9] CI: Skip windows/py312 tests --- .github/workflows/main.yaml | 3 +++ .github/workflows/pull_request.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index b82bc063..8b41deb6 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -25,6 +25,9 @@ jobs: os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] extra-args: ["", "--concat"] + exclude: + - os: windows-latest + python-version: 3.12 env: PLATFORM: ${{ matrix.os }} diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 6e030080..892f3a1e 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -16,6 +16,9 @@ jobs: os: ["ubuntu-latest", "macos-latest", "windows-latest"] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] extra-args: ["", "--concat"] + exclude: + - os: windows-latest + python-version: 3.12 env: PLATFORM: ${{ matrix.os }} From 3c52199e302111d0dcb5b236b5ec193e23ebda27 Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Mon, 23 Oct 2023 15:23:18 +0100 Subject: [PATCH 7/9] CI: All dependencies should already be installed in test env, so don't perform an isolated build --- .ci/build_dev_indexed_gzip.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/build_dev_indexed_gzip.sh b/.ci/build_dev_indexed_gzip.sh index bbf5778f..06f92c2e 100755 --- a/.ci/build_dev_indexed_gzip.sh +++ b/.ci/build_dev_indexed_gzip.sh @@ -13,4 +13,4 @@ source $thisdir/activate_env.sh "$envdir" # modules - see setup.py export INDEXED_GZIP_TESTING=1 -pip install -e . +pip install --no-build-isolation --editable . From b620bf284432e1c097a91dff57401e501c0eec3d Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Mon, 23 Oct 2023 16:00:39 +0100 Subject: [PATCH 8/9] TEST: Address test warnings --- indexed_gzip/tests/__init__.py | 2 +- indexed_gzip/tests/ctest_indexed_gzip.pyx | 22 +++++++++++----------- indexed_gzip/tests/ctest_zran.pyx | 6 +++--- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/indexed_gzip/tests/__init__.py b/indexed_gzip/tests/__init__.py index 25d90966..f5822fac 100644 --- a/indexed_gzip/tests/__init__.py +++ b/indexed_gzip/tests/__init__.py @@ -77,7 +77,7 @@ def compress_with_gzip_module(): if len(data) == 0: break with open(outfile, 'ab') as outf: - gzip.GzipFile(fileobj=outf).write(data) + gzip.GzipFile(fileobj=outf, mode='ab').write(data) def compress_with_gzip_command(): diff --git a/indexed_gzip/tests/ctest_indexed_gzip.pyx b/indexed_gzip/tests/ctest_indexed_gzip.pyx index abf963e0..dd8d9b9f 100644 --- a/indexed_gzip/tests/ctest_indexed_gzip.pyx +++ b/indexed_gzip/tests/ctest_indexed_gzip.pyx @@ -68,7 +68,7 @@ def test_open_close(testfile, nelems, seed, drop): assert not f.closed - element = np.random.randint(0, nelems, 1) + element = np.random.randint(0, nelems, 1)[0] readval = read_element(f, element) assert readval == element @@ -90,7 +90,7 @@ def test_open_function(testfile, nelems): f1 = igzip.IndexedGzipFile(testfile) f2 = igzip.open( testfile) - element = np.random.randint(0, nelems, 1) + element = np.random.randint(0, nelems, 1)[0] readval1 = read_element(f1, element) readval2 = read_element(f2, element) @@ -106,7 +106,7 @@ def test_open_close_ctxmanager(testfile, nelems, seed, drop): with igzip._IndexedGzipFile(filename=testfile, drop_handles=drop) as f: - element = np.random.randint(0, nelems, 1) + element = np.random.randint(0, nelems, 1)[0] readval = read_element(f, element) assert readval == element @@ -262,7 +262,7 @@ def test_create_from_open_handle(testfile, nelems, seed, drop, file_like_object) assert gzf.fileobj() is f assert not gzf.drop_handles - element = np.random.randint(0, nelems, 1) + element = np.random.randint(0, nelems, 1)[0] readval = read_element(gzf, element) gzf.close() @@ -291,7 +291,7 @@ def test_accept_filename_or_fileobj(testfile, nelems): gzf2 = igzip._IndexedGzipFile(f) gzf3 = igzip._IndexedGzipFile(fileobj=BytesIO(open(testfile, 'rb').read())) - element = np.random.randint(0, nelems, 1) + element = np.random.randint(0, nelems, 1)[0] readval1 = read_element(gzf1, element) readval2 = read_element(gzf2, element) readval3 = read_element(gzf3, element) @@ -329,7 +329,7 @@ def test_prioritize_fd_over_f(testfile, nelems): f.read = error_fn # If the file-like object were directly used by zran.c, reading would raise an error. gzf = igzip._IndexedGzipFile(fileobj=f) - element = np.random.randint(0, nelems, 1) + element = np.random.randint(0, nelems, 1)[0] readval = read_element(gzf, element) assert readval == element @@ -353,7 +353,7 @@ def test_handles_not_dropped(testfile, nelems, seed): # doesn't change across reads for i in range(5): - element = np.random.randint(0, nelems, 1) + element = np.random.randint(0, nelems, 1)[0] readval = read_element(f, element) assert readval == element @@ -367,7 +367,7 @@ def test_handles_not_dropped(testfile, nelems, seed): for i in range(5): - element = np.random.randint(0, nelems, 1) + element = np.random.randint(0, nelems, 1)[0] readval = read_element(gzf, element) assert readval == element @@ -413,7 +413,7 @@ def test_manual_build(): f.build_full_index() for i in range(5): - element = np.random.randint(0, nelems, 1) + element = np.random.randint(0, nelems, 1)[0] readval = read_element(f, element) assert readval == element @@ -925,7 +925,7 @@ def test_build_index_from_unseekable(): # make a test file data = np.arange(524288, dtype=np.uint64) with gzip.open(fname, 'wb') as f: - f.write(data.tostring()) + f.write(data.tobytes()) # Test creating the index when file is unseekable, # then using the index when file is seekable. @@ -967,7 +967,7 @@ def test_wrapper_class(): data = np.arange(65536, dtype=np.uint64) with gzip.open(fname, 'wb') as f: - f.write(data.tostring()) + f.write(data.tobytes()) with igzip.IndexedGzipFile(fname, drop_handles=False) as f: diff --git a/indexed_gzip/tests/ctest_zran.pyx b/indexed_gzip/tests/ctest_zran.pyx index 55732cf2..f9304bde 100644 --- a/indexed_gzip/tests/ctest_zran.pyx +++ b/indexed_gzip/tests/ctest_zran.pyx @@ -181,7 +181,7 @@ cdef class ReadBuffer: zeros = np.zeros(min(towrite, 134217728), dtype=np.uint8) towrite -= len(zeros) - os.write(fd, zeros.tostring()) + os.write(fd, zeros.tobytes()) th = threading.Thread(target=initmem) th.start() @@ -1133,7 +1133,7 @@ def test_export_import_no_points(no_fds): with tempdir(): with gzip.open('data.gz', 'wb') as f: - f.write(data.tostring()) + f.write(data.tobytes()) with open('data.gz', 'rb') as pyfid: cfid = fdopen(pyfid.fileno(), 'rb') @@ -1187,7 +1187,7 @@ def test_export_import_format_v0(): with tempdir(): with gzip.open('data.gz', 'wb') as f: - f.write(data.tostring()) + f.write(data.tobytes()) with open('data.gz', 'rb') as pyfid: cfid = fdopen(pyfid.fileno(), 'rb') From 359844ad23067415b2552b71af7ea197a797a3fa Mon Sep 17 00:00:00 2001 From: Paul McCarthy Date: Mon, 23 Oct 2023 16:19:48 +0100 Subject: [PATCH 9/9] TEST: Explicitly delete file handle and remove file to (hopefully) avoid file-not-closed errors under Windows --- indexed_gzip/tests/__init__.py | 3 ++- indexed_gzip/tests/ctest_indexed_gzip.pyx | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/indexed_gzip/tests/__init__.py b/indexed_gzip/tests/__init__.py index f5822fac..2ccb0acb 100644 --- a/indexed_gzip/tests/__init__.py +++ b/indexed_gzip/tests/__init__.py @@ -20,7 +20,7 @@ def tempdir(): - """Returnsa context manager which creates and returns a temporary + """Returns a context manager which creates and returns a temporary directory, and then deletes it on exit. """ @@ -34,6 +34,7 @@ def __enter__(self): def __exit__(self, *a, **kwa): os.chdir(self.prevdir) + time.sleep(0.25) shutil.rmtree(self.tempdir) return ctx() diff --git a/indexed_gzip/tests/ctest_indexed_gzip.pyx b/indexed_gzip/tests/ctest_indexed_gzip.pyx index dd8d9b9f..d84ea2f4 100644 --- a/indexed_gzip/tests/ctest_indexed_gzip.pyx +++ b/indexed_gzip/tests/ctest_indexed_gzip.pyx @@ -493,6 +493,10 @@ def test_read_beyond_end(concat, drop): assert check_data_valid(data1, 0) assert check_data_valid(data2, 0) + del f + f = None + os.remove(testfile) + def test_seek(concat): with tempdir() as tdir: