[med-svn] [Git][med-team/cyvcf2][master] 7 commits: New upstream version 0.30.28
Andreas Tille (@tille)
gitlab at salsa.debian.org
Wed Feb 7 10:40:32 GMT 2024
Andreas Tille pushed to branch master at Debian Med / cyvcf2
Commits:
7c97cba8 by Andreas Tille at 2024-02-07T09:56:39+01:00
New upstream version 0.30.28
- - - - -
1c6721f6 by Andreas Tille at 2024-02-07T09:56:39+01:00
routine-update: New upstream version
- - - - -
c62c6f96 by Andreas Tille at 2024-02-07T09:56:39+01:00
Update upstream source from tag 'upstream/0.30.28'
Update to upstream version '0.30.28'
with Debian dir 2c6cbcd97d5d6a8aa3b547c8ea4c9bbbccd39069
- - - - -
07f5556f by Andreas Tille at 2024-02-07T11:15:12+01:00
Set CYVCF2_HTSLIB_MODE=EXTERNAL to build with Debian packaged htslib
- - - - -
94090a0b by Andreas Tille at 2024-02-07T11:33:08+01:00
Force copying tests dir into build_dir
- - - - -
3530f89b by Andreas Tille at 2024-02-07T11:35:55+01:00
Builds against libhts 19 from experimental
- - - - -
f957b105 by Andreas Tille at 2024-02-07T11:38:55+01:00
Upload to unstable
- - - - -
28 changed files:
- .github/workflows/build.yml
- .github/workflows/wheels.yml
- .gitignore
- CHANGES.md
- MANIFEST.in
- README.md
- + ci/Dockerfile.alpine.test
- ci/Dockerfile.test → ci/Dockerfile.slim.test
- ci/linux-deps
- − ci/osx-arm64-deps
- ci/osx-deps
- cyvcf2/__init__.py
- cyvcf2/cyvcf2.pxd
- cyvcf2/cyvcf2.pyx
- cyvcf2/helpers.c
- + cyvcf2/tests/multi-contig.bcf
- + cyvcf2/tests/multi-contig.bcf.csi
- + cyvcf2/tests/multi-contig.vcf.gz
- + cyvcf2/tests/multi-contig.vcf.gz.csi
- + cyvcf2/tests/multi-contig.vcf.gz.tbi
- cyvcf2/tests/test_reader.py
- debian/changelog
- debian/patches/i386.patch
- debian/patches/series
- debian/rules
- pyproject.toml
- requirements.txt
- setup.py
Changes:
=====================================
.github/workflows/build.yml
=====================================
@@ -3,14 +3,25 @@ name: Build
on: [push, pull_request]
jobs:
- docker-build-and-test:
- name: Docker Build and Test on ${{ matrix.platform }}
- runs-on: ubuntu-latest
+ docker-build:
+ name: Docker Run Test on ${{ matrix.platform }}-${{ matrix.python_tag_type }}
+ runs-on: ubuntu-22.04
strategy:
matrix:
platform:
- linux/amd64
- linux/arm64
+ python_tag_type:
+ - slim
+ - alpine
+ exclude:
+ # amd64 with glibc have full direct test
+ - platform: linux/amd64
+ python_tag_type: slim
+
+ # test alpine only on amd64
+ - platform: linux/arm64
+ python_tag_type: alpine
steps:
- uses: actions/checkout at v4
@@ -20,51 +31,184 @@ jobs:
uses: docker/setup-qemu-action at v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action at v3
- - name: Docker Build and Push
+
+ - name: Docker Build
uses: docker/build-push-action at v5
with:
- context: .
- file: ./ci/Dockerfile.test
+ context: .
+ file: ./ci/Dockerfile.${{ matrix.python_tag_type }}.test
platforms: ${{ matrix.platform }}
- tags: cyvcf2:test
+ tags: cyvcf2:${{ matrix.python_tag_type }}-test
push: false
load: true
- build-args: |
- PYTHON_VERSION=slim
+ build-args: |
+ PYTHON_VERSION=${{ matrix.python_tag_type }}
- name: Docker Run Tests
run: |
- docker run --rm --platform ${{ matrix.platform }} cyvcf2:test pytest --cov cyvcf2 --cov-report term-missing
+ docker run --rm --platform ${{ matrix.platform }} cyvcf2:${{ matrix.python_tag_type }}-test pytest --cov cyvcf2 --cov-report term-missing
build:
- name: Run tests on Python ${{ matrix.python-version }}
- runs-on: ubuntu-20.04
+ name: Run tests on Python ${{ matrix.python-version }} ${{ matrix.os }}
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os: [ubuntu-22.04, macos-12]
+ python-version:
+ ["pypy3.10", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+ exclude:
+ # Run only the latest versions on macOS and windows
+ - os: macos-12
+ python-version: "pypy3.10"
+ - os: macos-12
+ python-version: "3.7"
+ - os: macos-12
+ python-version: "3.8"
+ - os: macos-12
+ python-version: "3.9"
+ - os: macos-12
+ python-version: "3.10"
+ - os: macos-12
+ python-version: "3.11"
+
+ steps:
+ - uses: actions/checkout at v4
+ with:
+ submodules: recursive
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python at v5
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Set macOS env
+ if: runner.os == 'macOS'
+ run: |
+ # building options
+ echo "MACOSX_DEPLOYMENT_TARGET=10.9" >> "$GITHUB_ENV"
+ echo "ARCHFLAGS=-arch x86_64" >> "$GITHUB_ENV"
+
+ - name: Install Linux build prerequisites
+ if: runner.os == 'Linux'
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y --no-install-recommends libcurl4-openssl-dev zlib1g-dev libssl-dev liblzma-dev \
+ libbz2-dev libdeflate-dev
+
+ - name: Install macOS build prerequisites
+ if: runner.os == 'macOS'
+ run: |
+ brew install automake libdeflate
+
+ - name: Install
+ run: |
+ pip install -r requirements.txt
+ pip install pytest pytest-cov
+ CYVCF2_HTSLIB_CONFIGURE_OPTIONS="--enable-libcurl --enable-s3 --enable-lzma --enable-bz2 --with-libdeflate" \
+ CYTHONIZE=1 python setup.py build_ext -i
+
+ - name: Test
+ run: |
+ pytest --cov cyvcf2 --cov-report term-missing
+
+ windows_build:
+ name: Run tests on Python windows-2022 MSYS2 UCRT64
+ runs-on: windows-2022
+ defaults:
+ run:
+ shell: msys2 {0}
+
+ steps:
+ - uses: actions/checkout at v4
+ with:
+ submodules: recursive
+
+ - name: "Setup MSYS2"
+ uses: msys2/setup-msys2 at v2
+ with:
+ msystem: UCRT64
+ path-type: inherit
+ install: >-
+ mingw-w64-ucrt-x86_64-gcc
+ mingw-w64-ucrt-x86_64-make
+ mingw-w64-ucrt-x86_64-libdeflate
+ mingw-w64-ucrt-x86_64-xz
+ mingw-w64-ucrt-x86_64-curl
+ mingw-w64-ucrt-x86_64-zlib
+ mingw-w64-ucrt-x86_64-bzip2
+ mingw-w64-ucrt-x86_64-tools-git
+ mingw-w64-ucrt-x86_64-python-pkgconfig
+ mingw-w64-ucrt-x86_64-pkg-config
+ mingw-w64-ucrt-x86_64-ninja
+ mingw-w64-ucrt-x86_64-python
+ mingw-w64-ucrt-x86_64-python-pip
+ make
+ automake
+ autoconf
+ git
+
+ - name: Install Windows build prerequisites
+ run: |
+ cd htslib
+ autoreconf -i
+ ./configure --enable-libcurl --enable-s3 --enable-lzma --enable-bz2 --with-libdeflate
+ make
+ make install
+
+ - name: Install
+ run: |
+ pip install -r requirements.txt
+ pip install pytest pytest-cov
+ CYTHONIZE=1 python setup.py build_ext -i
+
+ - name: Test
+ run: |
+ pytest --cov cyvcf2 --cov-report term-missing
+
+ sdist:
+ runs-on: ubuntu-22.04
strategy:
matrix:
- python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+ python-version: ["3.12"]
steps:
- - uses: actions/checkout at v3
- with:
- submodules: true
- - name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python at v4
- with:
- python-version: ${{ matrix.python-version }}
- - name: Install
- run: |
- sudo apt-get update
- sudo apt-get install libcurl4-openssl-dev
- pip install -r requirements.txt
- pip install pytest pytest-cov
- git submodule update --init --recursive
- cd htslib
- autoheader && autoconf && autoreconf --install
- ./configure --enable-s3 --disable-lzma --disable-bz2
- make
- cd ..
- CYTHONIZE=1 python setup.py build_ext -i
- - name: Test
- run: |
- pytest --cov cyvcf2 --cov-report term-missing
+ - uses: actions/checkout at v4
+ with:
+ submodules: recursive
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python at v5
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Install sdist prerequisites
+ run: |
+ pip install -r requirements.txt
+
+ - name: Create source distribution
+ run: CYTHONIZE=1 python setup.py sdist
+
+ - name: Install Linux build prerequisites
+ if: runner.os == 'Linux'
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y --no-install-recommends libcurl4-openssl-dev zlib1g-dev libssl-dev liblzma-dev \
+ libbz2-dev libdeflate-dev
+ - name: Build (via sdist tarball)
+ run: pip install --verbose --no-deps --no-binary='cyvcf2' cyvcf2-*.tar.gz
+ working-directory: dist
+
+ - name: Install test prerequisites
+ run: |
+ pip install pytest pytest-cov
+
+ - name: Test
+ run: |
+ pytest --import-mode importlib --cov cyvcf2 --cov-report term-missing
+
+ - name: Upload sdist tarball
+ if: runner.os == 'Linux'
+ uses: actions/upload-artifact at v4
+ with:
+ name: sdist
+ path: dist/cyvcf2-*.tar.gz
+ retention-days: 7
=====================================
.github/workflows/wheels.yml
=====================================
@@ -5,138 +5,141 @@ on:
branches:
- main
tags:
- - 'v*.*.*'
+ - "v*.*.*"
workflow_dispatch:
inputs:
debug_enabled:
- description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+ description: "Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)"
required: false
default: false
schedule:
# run weekly on a Monday
- - cron: '0 0 * * 1'
+ - cron: "0 0 * * 1"
jobs:
build_wheels:
- name: Build wheels on ${{ matrix.os }}
- runs-on: ${{ matrix.os }}
+ name: Build wheels for ${{ matrix.python-version }}-${{ matrix.buildplat[1] }}
+ runs-on: ${{ matrix.buildplat[0] }}
strategy:
matrix:
- os: [ubuntu-20.04, macos-latest]
+ buildplat:
+ - [ubuntu-22.04, manylinux_x86_64]
+ - [ubuntu-22.04, musllinux_x86_64]
+ - [ubuntu-22.04, manylinux_aarch64]
+ - [ubuntu-22.04, musllinux_aarch64]
+ - [macos-12, macosx_x86_64]
+ - [macos-12, macosx_arm64]
+ python-version: [pp310, cp37, cp38, cp39, cp310, cp311, cp312]
+ exclude:
+ # pp310, cp37, cp38 on musllinux is not support
+ # cp39, cp310 on musllinux_aarch64, wheel building may hangup, ignore it
+ - buildplat: [ubuntu-22.04, musllinux_x86_64]
+ python-version: cp37
+ - buildplat: [ubuntu-22.04, musllinux_x86_64]
+ python-version: cp38
+ - buildplat: [ubuntu-22.04, musllinux_x86_64]
+ python-version: pp310
+ - buildplat: [ubuntu-22.04, musllinux_aarch64]
+ python-version: cp37
+ - buildplat: [ubuntu-22.04, musllinux_aarch64]
+ python-version: cp38
+ - buildplat: [ubuntu-22.04, musllinux_aarch64]
+ python-version: cp39
+ - buildplat: [ubuntu-22.04, musllinux_aarch64]
+ python-version: cp310
+ - buildplat: [ubuntu-22.04, musllinux_aarch64]
+ python-version: pp310
+ # cp37, pp310 on macos arm64 is not supported
+ - buildplat: [macos-12, macosx_arm64]
+ python-version: cp37
+ - buildplat: [macos-12, macosx_arm64]
+ python-version: pp310
steps:
- - uses: actions/checkout at v3
- with:
- submodules: true
- - uses: actions/setup-python at v4
- name: Install Python
- with:
- python-version: "3.12"
-
- - name: Install cibuildwheel
- run: |
- python -m pip install -U cibuildwheel
-
- - name: Build wheels for Linux
- if: matrix.os == 'ubuntu-20.04'
- run: |
- python -m cibuildwheel --output-dir wheelhouse
- env:
- CIBW_SKIP: "pp* *i686* *musllinux*"
- CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
- CIBW_BEFORE_BUILD_LINUX: "{project}/ci/linux-deps"
- CIBW_TEST_COMMAND: "{project}/ci/test"
- CIBW_ENVIRONMENT: "CYTHONIZE=1 LIBDEFLATE=1 LDFLAGS='-L/usr/lib64/openssl11' CPPFLAGS='-I/usr/include/openssl11' C_INCLUDE_PATH='/root/include' LIBRARY_PATH='/root/lib'"
- CIBW_REPAIR_WHEEL_COMMAND_LINUX: LD_LIBRARY_PATH='/root/lib' auditwheel repair -w {dest_dir} {wheel}
-
- - name: Build wheels for Mac OS x86
- if: matrix.os == 'macos-latest'
- run: |
- python -m cibuildwheel --output-dir wheelhouse
- env:
- CIBW_SKIP: "pp* *i686*"
- CIBW_ARCHS_MACOS: "x86_64"
- CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
- CIBW_BEFORE_BUILD_MACOS: "{project}/ci/osx-deps"
- CIBW_TEST_COMMAND: "{project}/ci/test"
- CIBW_ENVIRONMENT: "CYTHONIZE=1 LIBDEFLATE=1 C_INCLUDE_PATH='/usr/local/include' LIBRARY_PATH='/usr/local/lib'"
- # https://cibuildwheel.readthedocs.io/en/stable/faq/#macos-passing-dyld_library_path-to-delocate
- CIBW_REPAIR_WHEEL_COMMAND_MACOS: >
- DYLD_LIBRARY_PATH=/usr/local/lib delocate-listdeps {wheel} &&
- DYLD_LIBRARY_PATH=/usr/local/lib delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}
- LDFLAGS: "-L/usr/local/opt/openssl at 1.1/lib"
- CPPFLAGS: "-I/usr/local/opt/openssl at 1.1/include"
- PKG_CONFIG_PATH: "/usr/local/opt/openssl at 1.1/lib/pkgconfig"
-
- - name: Build wheels for Mac OS arm64
- # don't build with libdeflate, see https://github.com/brentp/cyvcf2/issues/252
- if: matrix.os == 'macos-latest'
- run: |
- python -m cibuildwheel --output-dir wheelhouse
- env:
- CIBW_SKIP: "pp* *i686*"
- CIBW_ARCHS_MACOS: "arm64"
- CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
- CIBW_BEFORE_BUILD_MACOS: "{project}/ci/osx-arm64-deps"
- CIBW_TEST_COMMAND: "{project}/ci/test"
- CIBW_TEST_SKIP: "*-macosx_arm64"
- CIBW_ENVIRONMENT: "CYTHONIZE=1 C_INCLUDE_PATH='/usr/local/include' LIBRARY_PATH='/usr/local/lib'"
- # https://cibuildwheel.readthedocs.io/en/stable/faq/#macos-passing-dyld_library_path-to-delocate
- CIBW_REPAIR_WHEEL_COMMAND_MACOS: >
- DYLD_LIBRARY_PATH=/usr/local/lib delocate-listdeps {wheel} &&
- DYLD_LIBRARY_PATH=/usr/local/lib delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}
- LDFLAGS: "-L/usr/local/opt/openssl at 1.1/lib"
- CPPFLAGS: "-I/usr/local/opt/openssl at 1.1/include"
- PKG_CONFIG_PATH: "/usr/local/opt/openssl at 1.1/lib/pkgconfig"
-
- # Enable tmate debugging of manually-triggered workflows if the input option was provided
- - name: Setup tmate session
- if: ${{ always() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true' }}
- uses: mxschmitt/action-tmate at v3
-
- - uses: actions/upload-artifact at v2
- with:
- path: ./wheelhouse/*.whl
+ - uses: actions/checkout at v4
+ with:
+ submodules: recursive
+
+ - name: Set up QEMU
+ if: runner.os == 'Linux'
+ uses: docker/setup-qemu-action at v3
+
+ - name: Build wheels
+ uses: pypa/cibuildwheel at v2.16.4
+ with:
+ package-dir: .
+ output-dir: wheelhouse
+ config-file: "{package}/pyproject.toml"
+ env:
+ # select
+ CIBW_BUILD: ${{ matrix.python-version }}-${{ matrix.buildplat[1] }}
+
+ # linux
+ CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+ # manylinux2014 can't build on aarch64
+ CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
+ CIBW_MUSLLINUX_X86_64_IMAGE: musllinux_1_2
+ CIBW_MUSLLINUX_AARCH64_IMAGE: musllinux_1_2
+ CIBW_ARCHS_LINUX: auto64 aarch64
+ CIBW_BEFORE_BUILD_LINUX: "{project}/ci/linux-deps"
+ CIBW_REPAIR_WHEEL_COMMAND_LINUX: 'LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib64" && auditwheel repair -w {dest_dir} {wheel}'
+
+ # macos
+ CIBW_ARCHS_MACOS: auto64 arm64
+ CIBW_BEFORE_BUILD_MACOS: "{project}/ci/osx-deps"
+
+ # build
+ CIBW_ENVIRONMENT: >-
+ CYVCF2_HTSLIB_CONFIGURE_OPTIONS="--enable-libcurl --enable-s3 --enable-lzma --enable-bz2 --with-libdeflate"
+ CYTHONIZE=1
+ CIBW_TEST_COMMAND: "{project}/ci/test"
+ # macOS build arm64 on x86_64 with cross-compiler, not support test
+ CIBW_TEST_SKIP: "*-macosx_arm64"
+
+ # Enable tmate debugging of manually-triggered workflows if the input option was provided
+ - name: Setup tmate session
+ if: ${{ always() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true' }}
+ uses: mxschmitt/action-tmate at v3
+
+ - uses: actions/upload-artifact at v4
+ with:
+ name: artifact-${{ matrix.python-version }}-${{ matrix.buildplat[1] }}
+ path: ./wheelhouse/*.whl
build_sdist:
name: Build source distribution
- runs-on: ubuntu-latest
+ runs-on: ubuntu-22.04
steps:
- - uses: actions/checkout at v2
-
- - uses: actions/setup-python at v4
+ - uses: actions/checkout at v4
+ with:
+ submodules: recursive
+ - uses: actions/setup-python at v5
name: Install Python
with:
python-version: "3.12"
- name: Install dependencies
run: |
- sudo apt-get update
- sudo apt-get install libcurl4-openssl-dev libbz2-dev liblzma-dev libssl-dev
- git submodule update --init --recursive
- cd htslib
- autoheader && autoconf && autoreconf --install
- ./configure --enable-libcurl --enable-s3 --enable-lzma --enable-bz2
- make
- cd ..
pip install -r requirements.txt
- name: Build sdist
run: CYTHONIZE=1 python setup.py sdist
- - uses: actions/upload-artifact at v2
+ - uses: actions/upload-artifact at v4
with:
+ name: artifact-sdist
path: dist/*.tar.gz
upload_pypi:
needs: [build_wheels, build_sdist]
- runs-on: ubuntu-latest
+ runs-on: ubuntu-22.04
# upload to PyPI on every tag starting with 'v'
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
steps:
- - uses: actions/download-artifact at v2
+ - uses: actions/download-artifact at v4
with:
- name: artifact
+ pattern: artifact-*
+ merge-multiple: true
path: dist
- uses: pypa/gh-action-pypi-publish at release/v1
=====================================
.gitignore
=====================================
@@ -19,3 +19,4 @@ _templates
setup-requires/*
.cache/v/cache/lastfailed
.idea
+**/__pycache__
=====================================
CHANGES.md
=====================================
@@ -1,3 +1,8 @@
+# v0.30.27
+
++ support num_records using index stats (#295 from @jeromekelleher)
++ don't remove htslib/config.status (#296)
+
# v0.30.25
+ bump to get new release for CI/wheel fixes. thanks to @graphenn (#286) and @oyvinev (#297)
=====================================
MANIFEST.in
=====================================
@@ -1,12 +1,18 @@
-include cyvcf2/tests/*
include LICENSE
+include *.md
include cyvcf2/*.pyx
include cyvcf2/*.pxd
-include cyvcf2/*.c
-include cyvcf2/*.h
-include *.md
-include htslib/*.c
-include htslib/*.h
-include htslib/htslib/*.h
-include htslib/cram/*.c
-include htslib/cram/*.h
+include cyvcf2/*.[ch]
+
+# exclude tests from pypi because no data file
+prune cyvcf2/tests
+
+# htslib
+include htslib/LICENSE htslib/README
+recursive-include htslib *.[ch]
+exclude htslib/*config*.h
+
+include htslib/configure.ac htslib/m4/*.m4 htslib/*.in
+include htslib/configure htslib/version.sh
+include htslib/Makefile htslib/*.mk
+exclude htslib/config.mk htslib/htscodecs.mk
=====================================
README.md
=====================================
@@ -14,7 +14,7 @@ If you use cyvcf2, please cite the [paper](https://academic.oup.com/bioinformati
Fast python **(2 and 3)** parsing of VCF and BCF including region-queries.
-[](https://github.com/brentp/cyvcf2/actions?query=workflow%3ABuild)
+[](https://github.com/brentp/cyvcf2/actions/workflows/build.yml)
cyvcf2 is a cython wrapper around [htslib](https://github.com/samtools/htslib) built for fast parsing of [Variant Call Format](https://en.m.wikipedia.org/wiki/Variant_Call_Format) (VCF) files.
@@ -70,24 +70,35 @@ for v in vcf('11:435345-556565'):
Installation
============
-## pip (assuming you have htslib < 1.10 installed)
+## pip with bundled htslib
```
pip install cyvcf2
```
+## pip with system htslib
+
+Assuming you have already built and installed htslib version 1.12 or higher.
+```
+CYVCF2_HTSLIB_MODE=EXTERNAL pip install --no-binary cyvcf2 cyvcf2
+```
+
+## windows (experimental, only test on MSYS2)
+
+Assuming you have already built and installed htslib.
+```
+SETUPTOOLS_USE_DISTUTILS=stdlib pip install cyvcf2
+```
+
## github (building htslib and cyvcf2 from source)
```
git clone --recursive https://github.com/brentp/cyvcf2
-cd cyvcf2/htslib
-autoheader
-autoconf
-./configure --enable-libcurl
-make
-
-cd ..
pip install -r requirements.txt
-CYTHONIZE=1 pip install -e .
+# sometimes it can be required to remove old files:
+# python setup.py clean_ext
+CYVCF2_HTSLIB_MODE=BUILTIN CYTHONIZE=1 python setup.py install
+# or to use a system htslib.so
+CYVCF2_HTSLIB_MODE=EXTERNAL python setup.py install
```
On **OSX**, using brew, you may have to set the following as indicated by the brew install:
=====================================
ci/Dockerfile.alpine.test
=====================================
@@ -0,0 +1,15 @@
+ARG PYTHON_VERSION=alpine
+
+FROM python:${PYTHON_VERSION}
+
+WORKDIR /workspace
+
+RUN apk add --no-cache build-base autoconf automake git xz-dev curl-dev libdeflate-dev bzip2-dev
+
+COPY . .
+
+RUN pip install -r requirements.txt && pip install pytest pytest-cov
+
+# build cyvcf2
+RUN CYVCF2_HTSLIB_CONFIGURE_OPTIONS="--enable-libcurl --enable-s3 --enable-lzma --enable-bz2 --with-libdeflate" \
+ CYTHONIZE=1 python setup.py build_ext -i
=====================================
ci/Dockerfile.test → ci/Dockerfile.slim.test
=====================================
@@ -4,17 +4,14 @@ FROM python:${PYTHON_VERSION}
WORKDIR /workspace
-RUN apt-get update && apt-get install --no-install-recommends -y autoconf automake gcc \
- libcurl4-openssl-dev make git libc6-dev zlib1g-dev libssl-dev liblzma-dev libbz2-dev && \
+RUN apt-get update && apt-get install --no-install-recommends -y autoconf automake gcc libcurl4-openssl-dev \
+ make git libc6-dev zlib1g-dev libssl-dev liblzma-dev libbz2-dev libdeflate-dev && \
rm -rf /var/lib/apt/lists/*
COPY . .
-# build htslib
-RUN cd htslib && autoheader && autoconf && autoreconf --install && \
- ./configure --enable-s3 --disable-lzma --disable-bz2 && make
-
RUN pip install -r requirements.txt && pip install pytest pytest-cov
# build cyvcf2
-RUN CYTHONIZE=1 python setup.py build_ext -i
+RUN CYVCF2_HTSLIB_CONFIGURE_OPTIONS="--enable-libcurl --enable-s3 --enable-lzma --enable-bz2 --with-libdeflate" \
+ CYTHONIZE=1 python setup.py build_ext -i
=====================================
ci/linux-deps
=====================================
@@ -1,44 +1,43 @@
#!/bin/bash
-set -e
-
-yum install -y openssl11-devel
-yum install -y libffi libffi-devel zlib-devel
-yum install -y bzip2-devel bzip2-libs xz-devel xz-libs
-
-git submodule init
-git submodule update
-git submodule update --init --recursive
-
-curl -L -O https://www.libssh2.org/download/libssh2-1.9.0.tar.gz
-tar xzf libssh2-1.9.0.tar.gz
-cd libssh2-1.9.0
-./configure --prefix=/root
-make
-make install
-cd ..
-rm -rf libssh2-1.9.0
-
-curl -L -O https://curl.se/download/curl-7.73.0.tar.gz
-tar xzf curl-7.73.0.tar.gz
-cd curl-7.73.0
-./configure --with-libssh2 --prefix=/root
-make
-make install
-cd ..
-rm -rf curl-7.73.0
-
-curl -L -o libdeflate-v1.8.tar.gz https://github.com/ebiggers/libdeflate/archive/refs/tags/v1.8.tar.gz
-tar xzf libdeflate-v1.8.tar.gz
-cd libdeflate-1.8
-make
-make install PREFIX=/root
-cd ..
-
-cd htslib
-# configure fails with autoconf 2.71 (in /usr/local/bin), so use 2.69 (in /usr/bin)
-/usr/bin/autoconf -V
-/usr/bin/autoheader
-/usr/bin/autoconf
-./configure --enable-libcurl --enable-s3 --enable-lzma --enable-bz2
-make
+# Configure the libraries needed to build wheel packages on linux.
+# This script is designed to be used by cibuildwheel as CIBW_BEFORE_ALL_LINUX
+
+set -euo pipefail
+
+# dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source /etc/os-release
+
+echo "manylinux image woking on $ID"
+
+# Install cyvcf2 development files.
+case "$ID" in
+almalinux)
+ dnf install -y bzip2-devel xz-devel libcurl-devel openssl-devel openblas-devel epel-release
+
+ # packages at epel
+ dnf install -y libdeflate-devel
+ ;;
+
+alpine)
+ apk add --no-cache xz-dev curl-dev libdeflate-dev
+ ;;
+
+centos)
+ yum install -y bzip2-devel xz-devel libcurl-devel openssl-devel
+
+ LIBDEFLATE_VERSION=1.19
+ curl -L -o libdeflate-v"$LIBDEFLATE_VERSION".tar.gz https://github.com/ebiggers/libdeflate/archive/refs/tags/v"$LIBDEFLATE_VERSION".tar.gz
+ tar xzf libdeflate-v"$LIBDEFLATE_VERSION".tar.gz
+ cd libdeflate-"$LIBDEFLATE_VERSION"
+ cmake -B build && cmake --build build
+ cd ./build/ && make install
+ cd ../..
+ ;;
+
+*)
+ echo "$0: unexpected Linux distribution: '$ID'" >&2
+ exit 1
+ ;;
+esac
=====================================
ci/osx-arm64-deps deleted
=====================================
@@ -1,19 +0,0 @@
-#!/bin/bash
-
-set -e
-
-git submodule init
-git submodule update
-git submodule update --init --recursive
-
-# configure fails with autoconf 2.71, so downgrade
-# see https://github.com/asdf-vm/asdf-erlang/issues/195
-brew install autoconf at 2.69 && \
-brew link --overwrite autoconf at 2.69 && \
-autoconf -V
-
-cd htslib
-autoheader
-autoconf
-./configure --enable-libcurl --enable-s3 --enable-lzma --enable-bz2 --without-libdeflate
-make
=====================================
ci/osx-deps
=====================================
@@ -1,26 +1,26 @@
#!/bin/bash
-set -e
+set -euo pipefail
-git submodule init
-git submodule update
-git submodule update --init --recursive
+export DYLD_LIBRARY_PATH=/usr/local/lib
+# same with python
+export MACOSX_DEPLOYMENT_TARGET=10.9
-# configure fails with autoconf 2.71, so downgrade
-# see https://github.com/asdf-vm/asdf-erlang/issues/195
-brew install autoconf at 2.69 && \
-brew link --overwrite autoconf at 2.69 && \
-autoconf -V
+brew install automake
+brew unlink xz
-curl -L -o libdeflate-v1.8.tar.gz https://github.com/ebiggers/libdeflate/archive/refs/tags/v1.8.tar.gz
-tar xzf libdeflate-v1.8.tar.gz
-cd libdeflate-1.8
-make
-make install
-cd ..
+# build liblzma and libdelfate for muti arch
-cd htslib
-autoheader
-autoconf
-./configure --enable-libcurl --enable-s3 --enable-lzma --enable-bz2
-make
+XZ_VERSION=5.4.5
+curl -L -o xz-$XZ_VERSION.tar.gz "https://github.com/tukaani-project/xz/releases/download/v$XZ_VERSION/xz-$XZ_VERSION.tar.gz"
+tar -xf xz-$XZ_VERSION.tar.gz
+cd xz-$XZ_VERSION
+cmake -B build && cmake --build build
+cd ./build && make install
+
+LIBDEFLATE_VERSION=1.19
+curl -L -o libdeflate-v"$LIBDEFLATE_VERSION".tar.gz https://github.com/ebiggers/libdeflate/archive/refs/tags/v"$LIBDEFLATE_VERSION".tar.gz
+tar xzf libdeflate-v"$LIBDEFLATE_VERSION".tar.gz
+cd libdeflate-"$LIBDEFLATE_VERSION"
+cmake -B build && cmake --build build
+cd ./build && make install
=====================================
cyvcf2/__init__.py
=====================================
@@ -2,4 +2,4 @@ from .cyvcf2 import (VCF, Variant, Writer, r_ as r_unphased, par_relatedness,
par_het)
Reader = VCFReader = VCF
-__version__ = "0.30.25"
+__version__ = "0.30.28"
=====================================
cyvcf2/cyvcf2.pxd
=====================================
@@ -1,8 +1,11 @@
-from libc.stdint cimport int64_t, int32_t, uint32_t, int8_t, int16_t, uint8_t
+from libc.stdint cimport int64_t, uint64_t, int32_t, uint32_t, int8_t, int16_t, uint8_t
import numpy as np
cimport numpy as np
np.import_array()
+cdef extern from "string.h":
+ void* memcpy(void* dest, const void* src, size_t n)
+
cdef extern from "relatedness.h":
int related(int *gt_types, double *asum, int32_t *N, int32_t *ibs0,
int32_t *ibs2, int32_t n_samples)
@@ -67,15 +70,23 @@ cdef extern from "htslib/hts.h":
hts_idx_t *bcf_index_load(char *fn)
hts_idx_t *hts_idx_load2(const char *fn, const char *fnidx);
+ int hts_idx_nseq(const hts_idx_t *idx);
+ int hts_idx_get_stat(const hts_idx_t* idx, int tid, uint64_t* mapped,
+ uint64_t* unmapped);
#int hts_itr_next(BGZF *fp, hts_itr_t *iter, void *r, void *data);
void hts_itr_destroy(hts_itr_t *iter);
void hts_idx_destroy(hts_idx_t *idx);
cdef extern from "htslib/tbx.h":
+ ctypedef struct tbx_conf_t:
+ pass
+ # Expose details of tbx_t so that we can access the idx field
ctypedef struct tbx_t:
- pass
+ tbx_conf_t conf
+ hts_idx_t *idx
+ void *dict
tbx_t *tbx_index_load(const char *fn);
tbx_t *tbx_index_load2(const char *fn, const char *fnidx);
=====================================
cyvcf2/cyvcf2.pyx
=====================================
@@ -124,6 +124,24 @@ def r_(int32_t[::view.contiguous] a_gts, int32_t[::view.contiguous] b_gts, float
return r_unphased(&a_gts[0], &b_gts[0], f, n_samples)
+cdef char* pystring_to_cstring(str py_string):
+ # Convert Python string to byte string using UTF-8 encoding
+ cdef bytes byte_string = py_string.encode('utf-8') # Convert Python string to bytes
+ # Convert Python bytes to a C-compatible pointer
+ cdef const char* byte_string_ptr = byte_string
+
+ # Allocate memory for C string with size equal to byte string length plus null terminator
+ cdef size_t length = len(byte_string) + 1 # Plus 1 for the null terminator
+ cdef char* c_string = <char*>stdlib.malloc(length)
+ # return NULL if memory allocation fails
+ if c_string == NULL:
+ return c_string
+ # Copy byte string to C string using memcpy and add null terminator
+ memcpy(c_string, byte_string_ptr, length - 1) # Use memcpy to copy bytes to C string
+ c_string[length - 1] = b'\0' # Add null terminator to the end of the C string
+ return c_string
+
+
cdef set_constants(VCF v):
v.HOM_REF = 0
v.HET = 1
@@ -199,12 +217,14 @@ cdef class HTSFile:
"%s is not valid text_format or binary_format (format: %s mode: %s)" % (fname, self.hts.format.format, mode)
)
- def close(self):
+ cdef _c_close(self):
if self.hts != NULL:
if self.from_path:
hts_close(self.hts)
self.hts = NULL
+ def close(self):
+ self._c_close()
cdef class VCF(HTSFile):
"""
@@ -385,6 +405,14 @@ cdef class VCF(HTSFile):
raise Exception("unable to update to header")
def set_index(self, index_path=""):
+ # Clear any existing indexes
+ if self.idx != NULL:
+ tbx_destroy(self.idx)
+ self.idx = NULL
+ if self.hidx != NULL:
+ hts_idx_destroy(self.hidx)
+ self.hidx = NULL
+
if index_path.endswith(".tbi"):
self.idx = tbx_index_load2(to_bytes(self.fname), to_bytes(index_path))
if self.idx != NULL:
@@ -567,7 +595,9 @@ cdef class VCF(HTSFile):
if self.hdr != NULL:
bcf_hdr_destroy(self.hdr)
self.hdr = NULL
- self.close()
+
+ # Shouldn't call Python-space functions here
+ self._c_close()
if self.idx != NULL:
tbx_destroy(self.idx)
if self.hidx != NULL:
@@ -626,28 +656,73 @@ cdef class VCF(HTSFile):
stdlib.free(sls)
return self._seqlens
+ cdef _open_index(self):
+ """
+ Try to open an index, if not open already.
+ """
+ if self.hidx == NULL and self.idx == NULL:
+ if self.fname.decode(ENC).endswith(('.bcf', '.bcf.gz')):
+ self.hidx = bcf_index_load(self.fname)
+ else:
+ self.idx = tbx_index_load(to_bytes(self.fname))
+
property seqnames:
"list of chromosomes in the VCF"
def __get__(self):
if len(self._seqnames) > 0: return self._seqnames
- cdef char **cnames
+ cdef const char **cnames
cdef int i, n = 0
cnames = bcf_hdr_seqnames(self.hdr, &n)
- if n == 0 and self.fname.decode(ENC).endswith(('.bcf', '.bcf.gz')):
- if self.hidx == NULL:
- self.hidx = bcf_index_load(self.fname)
+ if n == 0:
+ self._open_index()
if self.hidx != NULL:
cnames = bcf_index_seqnames(self.hidx, self.hdr, &n)
- elif n == 0:
- if self.idx == NULL:
- self.idx = tbx_index_load(to_bytes(self.fname))
if self.idx !=NULL:
cnames = tbx_seqnames(self.idx, &n)
-
self._seqnames = [cnames[i].decode() for i in range(n)]
stdlib.free(cnames)
return self._seqnames
+ cdef _num_records(self):
+ cdef uint64_t total, records, v;
+ cdef int ret, tid, nseq;
+ cdef hts_idx_t *idx = NULL;
+
+ self._open_index()
+ if self.hidx != NULL:
+ idx = self.hidx
+ assert self.idx == NULL
+ if self.idx != NULL:
+ idx = self.idx.idx
+ assert self.hidx == NULL
+
+ if idx == NULL:
+ raise ValueError(
+ "File must be indexed to compute num_records (tip: use bcftools index)")
+
+ nseq = hts_idx_nseq(idx)
+ total = 0;
+ for tid in range(nseq):
+ # NOTE: the return value here doesn't seem to indicate an error
+ # condition, and correct values are computed when it returns < 0.
+ # bcftools index -n doesn't strictly check the output.
+ hts_idx_get_stat(idx, tid, &records, &v);
+ total += records
+ return total
+
+ property num_records:
+ """
+ The number of VCF records in the file, computed from the index.
+ If the file is not indexed (or an index has not been set using
+ ``set_index``) a ValueError is raised.
+
+ Note that incorrect values may be returned if a mismatched
+ index file (i.e., the index for a different VCF file) is used.
+ This is not detected as an error condition.
+ """
+ def __get__(self):
+ return self._num_records()
+
def plot_relatedness(self, riter):
import pandas as pd
from matplotlib import pyplot as plt
@@ -2379,19 +2454,29 @@ cdef class Writer(VCF):
def variant_from_string(self, variant_string):
cdef bcf1_t *b = bcf_init()
cdef kstring_t s
- tmp = to_bytes(variant_string)
- s.s = tmp
- s.l = len(variant_string)
- s.m = len(variant_string)
+
+ # vcf_parse may realloc s.s, so the memory must be created by malloc
+ s.s = pystring_to_cstring(variant_string)
+ if s.s == NULL:
+ bcf_destroy(b)
+ raise MemoryError("Failed to allocate memory")
+
+ # plus 1 for '\0' of cstring
+ s.l = len(variant_string) + 1
+ s.m = len(variant_string) + 1
+
ret = vcf_parse(&s, self.hdr, b)
if ret > 0:
bcf_destroy(b)
+ ks_free(&s)
raise Exception("error parsing:" + variant_string + " return value:" + ret)
var = newVariant(b, self)
if var.b.errcode == BCF_ERR_CTG_UNDEF:
self.add_to_header("##contig=<ID=%s>" % var.CHROM)
var.b.errcode = 0
+
+ ks_free(&s)
return var
def write_header(Writer self):
=====================================
cyvcf2/helpers.c
=====================================
@@ -4,10 +4,9 @@
int as_gts(int32_t *gts, int num_samples, int ploidy, int strict_gt, int HOM_ALT, int UNKNOWN) {
int j = 0, i, k;
- int missing= 0, found=0;
+ int missing= 0;
for (i = 0; i < ploidy * num_samples; i += ploidy){
missing = 0;
- found = 0;
for (k = 0; k < ploidy; k++) {
if bcf_gt_is_missing(gts[i+k]) {
missing += 1;
=====================================
cyvcf2/tests/multi-contig.bcf
=====================================
Binary files /dev/null and b/cyvcf2/tests/multi-contig.bcf differ
=====================================
cyvcf2/tests/multi-contig.bcf.csi
=====================================
Binary files /dev/null and b/cyvcf2/tests/multi-contig.bcf.csi differ
=====================================
cyvcf2/tests/multi-contig.vcf.gz
=====================================
Binary files /dev/null and b/cyvcf2/tests/multi-contig.vcf.gz differ
=====================================
cyvcf2/tests/multi-contig.vcf.gz.csi
=====================================
Binary files /dev/null and b/cyvcf2/tests/multi-contig.vcf.gz.csi differ
=====================================
cyvcf2/tests/multi-contig.vcf.gz.tbi
=====================================
Binary files /dev/null and b/cyvcf2/tests/multi-contig.vcf.gz.tbi differ
=====================================
cyvcf2/tests/test_reader.py
=====================================
@@ -1,5 +1,6 @@
from __future__ import print_function
import os.path
+import platform
import tempfile
import sys
import os
@@ -965,13 +966,20 @@ def test_alt_homozygous_gt():
assert v.gt_bases[0] == '<*:DEL>/<*:DEL>'
def test_write_missing_contig():
- input_vcf = VCF('{}/seg.vcf.gz'.format(HERE))
- output_vcf = Writer('/dev/null', input_vcf)
+ input_vcf = VCF("{}/seg.vcf.gz".format(HERE))
+ if platform.system() == "Windows":
+ output_vcf = "__o.vcf"
+ else:
+ output_vcf = "/dev/null"
+ output_vcf = Writer(output_vcf, input_vcf)
for v in input_vcf:
- v.genotypes = [[1,1,False]]
+ v.genotypes = [[1, 1, False]]
output_vcf.write_record(v)
output_vcf.close()
+ if platform.system() == "Windows":
+ os.unlink("__o.vcf")
+
def test_set_samples():
vcf = VCF(VCF_PATH)
assert len(vcf.samples) == 189, len(vcf.samples)
@@ -1000,9 +1008,14 @@ def test_issue44():
# "./." "." ".|." "0|0"
expected = [[-1, -1, False], [-1, False], [-1, -1, True], [0, 0, True]]
#print("", file=sys.stderr)
- for i, v in enumerate(VCF('__o.vcf')):
+
+ t = VCF('__o.vcf')
+ for i, v in enumerate(t):
#print(v.genotypes, file=sys.stderr)
assert v.genotypes == [expected[i]], (i, v.genotypes, expected[i])
+
+ # file should be closed before delete
+ t.close()
os.unlink("__o.vcf")
def test_id_field_updates():
@@ -1283,7 +1296,7 @@ def test_genotypes():
[0, 0, 1, 1],
[1, 1, 0, 0],
[1, 1, 0, 0],
- ]
+ ]
strict_exp_num = [x[:] for x in non_strict_exp_num]
strict_exp_num[1] = [0, 0, 2, 0] # both unknown
@@ -1323,3 +1336,77 @@ def test_issue17_no_gt():
with pytest.raises(Exception):
for v in vcf:
v.num_called # Used to give segmentation fault
+
+
+ at pytest.mark.parametrize("path", [
+ "test.vcf.gz",
+ "test-multiallelic-homozygous-alt.vcf.gz",
+ "test-strict-gt-option-flag.vcf.gz",
+ "test-strict-gt-option-flag.vcf.gz",
+ "multi-contig.vcf.gz",
+ "multi-contig.bcf",
+ "test.snpeff.bcf",
+ ])
+def test_num_records_indexed(path):
+ vcf = VCF(os.path.join(HERE, path))
+ n = len(list(vcf))
+ assert n == vcf.num_records
+ vcf = VCF(os.path.join(HERE, path))
+ assert n == vcf.num_records
+
+ at pytest.mark.parametrize("suffix", ["csi", "tbi"])
+def test_num_records_indexed_csi_tabix(suffix):
+ path = "multi-contig.vcf.gz"
+ index_file = os.path.join(HERE, "multi-contig.vcf.gz.{}".format(suffix))
+ vcf = VCF(os.path.join(HERE, path))
+ n = len(list(vcf))
+ # Explicitly set the index
+ vcf.set_index(index_file)
+ assert n == vcf.num_records
+ vcf = VCF(os.path.join(HERE, path))
+ vcf.set_index(index_file)
+ assert n == vcf.num_records
+
+def test_num_records_set_index_multiple_times():
+ path = os.path.join(HERE, "multi-contig.vcf.gz")
+ csi_index = path + ".csi"
+ tbi_index = path + ".tbi"
+ vcf = VCF(path)
+ n = len(list(vcf))
+ assert n == vcf.num_records
+ vcf.set_index(csi_index)
+ assert n == vcf.num_records
+
+ vcf = VCF(path)
+ assert n == vcf.num_records
+ vcf.set_index(tbi_index)
+ assert n == vcf.num_records
+
+ vcf = VCF(path)
+ vcf.set_index(csi_index)
+ assert n == vcf.num_records
+
+ vcf = VCF(path)
+ for _ in range(10):
+ vcf.set_index(csi_index)
+ assert n == vcf.num_records
+ vcf.set_index(tbi_index)
+ assert n == vcf.num_records
+
+def test_num_records_set_wrong_index():
+ path = os.path.join(HERE, "multi-contig.vcf.gz")
+ index = os.path.join(HERE, "test.vcf.gz.tbi")
+ vcf = VCF(path)
+ vcf.set_index(index)
+ # We compute the number of records from the index, and don't report an
+ # error
+ assert vcf.num_records == 115
+ assert vcf.num_records != len(list(vcf))
+
+ at pytest.mark.parametrize("path", [
+ "test-genotypes.vcf",
+ ])
+def test_num_records_no_index(path):
+ vcf = VCF(os.path.join(HERE, path))
+ with pytest.raises(ValueError, match="must be indexed"):
+ vcf.num_records
=====================================
debian/changelog
=====================================
@@ -1,3 +1,12 @@
+cyvcf2 (0.30.28-1) unstable; urgency=medium
+
+ * Team upload.
+ * New upstream version
+ Closes: #1058768
+ * Set CYVCF2_HTSLIB_MODE=EXTERNAL to build with Debian packaged htslib
+
+ -- Andreas Tille <tille at debian.org> Wed, 07 Feb 2024 11:37:34 +0100
+
cyvcf2 (0.30.25-1) unstable; urgency=medium
* Team upload.
=====================================
debian/patches/i386.patch
=====================================
@@ -1,7 +1,7 @@
Description: I386
--- a/cyvcf2/tests/test_reader.py
+++ b/cyvcf2/tests/test_reader.py
-@@ -15,6 +15,8 @@
+@@ -16,6 +16,8 @@ import pytest
from ..cyvcf2 import VCF, Variant, Writer
@@ -10,7 +10,7 @@ Description: I386
HERE = os.path.dirname(__file__)
VCF_PATH = os.path.join(HERE, "test.vcf.gz")
-@@ -1241,6 +1243,7 @@
+@@ -1254,6 +1256,7 @@ def test_no_seqlen():
with pytest.raises(AttributeError):
vcf.seqlens
=====================================
debian/patches/series
=====================================
@@ -1,2 +1,2 @@
-add_htslib_link_dependency.patch
+#add_htslib_link_dependency.patch
i386.patch
=====================================
debian/rules
=====================================
@@ -1,15 +1,17 @@
#!/usr/bin/make -f
-#export DH_VERBOSE = 1
+export DH_VERBOSE = 1
include /usr/share/dpkg/default.mk
export LC_ALL=C.UTF-8
export PYBUILD_NAME=cyvcf2
+PYBUILD_BEFORE_TEST=cp -a $(PYBUILD_NAME)/tests {build_dir}/$(PYBUILD_NAME);
export DEB_BUILD_MAINT_OPTIONS=hardening=+all
export CYTHONIZE=1
+export CYVCF2_HTSLIB_MODE=EXTERNAL
%:
dh $@ --buildsystem=pybuild
@@ -20,6 +22,10 @@ override_dh_install:
override_dh_auto_test:
ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
+ echo "$(PYBUILD_BEFORE_TEST) seems to be ignored - copying tests dir manually"
+ for tdir in .pybuild/cpython3_* ; do \
+ cp -a $(PYBUILD_NAME)/tests $${tdir}/build/$(PYBUILD_NAME) ; \
+ done
dh_auto_test
# clean up after testing
find . -name out.vcf -delete
=====================================
pyproject.toml
=====================================
@@ -1,2 +1,8 @@
[build-system]
-requires = ["setuptools", "wheel", "cython>=0.23.3", "oldest-supported-numpy"]
+requires = [
+ "setuptools",
+ "wheel",
+ "cython>=0.23.3",
+ 'oldest-supported-numpy; os_name != "nt"',
+ 'numpy; os_name == "nt"'
+]
=====================================
requirements.txt
=====================================
@@ -1,5 +1,6 @@
cython>=0.23.3
-numpy
coloredlogs
click
setuptools
+numpy; os_name == 'nt'
+oldest-supported-numpy; os_name != 'nt'
=====================================
setup.py
=====================================
@@ -1,13 +1,20 @@
-import os
+import ctypes
import glob
+import os
+import platform
+import shutil
import sys
import subprocess
-import platform
-from setuptools import setup, Extension
+from setuptools import setup, Extension, Command
+from setuptools.command.build_ext import build_ext
+from setuptools.command.sdist import sdist
if sys.version_info.major == 2 and sys.version_info.minor != 7:
- sys.stderr.write("ERROR: cyvcf2 is only for python 2.7 or greater you are running %d.%d\n", (sys.version_info.major, sys.version_info.minor))
+ sys.stderr.write(
+ "ERROR: cyvcf2 is only for python 2.7 or greater you are running %d.%d\n",
+ (sys.version_info.major, sys.version_info.minor),
+ )
sys.exit(1)
import numpy as np
@@ -18,15 +25,17 @@ def get_version():
import ast
with open(os.path.join("cyvcf2", "__init__.py"), "r") as init_file:
- module = ast.parse(init_file.read())
+ module = ast.parse(init_file.read())
- version = (ast.literal_eval(node.value) for node in ast.walk(module)
- if isinstance(node, ast.Assign)
- and node.targets[0].id == "__version__")
+ version = (
+ ast.literal_eval(node.value)
+ for node in ast.walk(module)
+ if isinstance(node, ast.Assign) and node.targets[0].id == "__version__"
+ )
try:
- return next(version)
+ return next(version)
except StopIteration:
- raise ValueError("version could not be located")
+ raise ValueError("version could not be located")
def no_cythonize(extensions, **_ignore):
@@ -41,30 +50,210 @@ def no_cythonize(extensions, **_ignore):
return extensions
+def check_libhts():
+ os_type = platform.system()
+ if os_type == "Linux":
+ lib_name = "libhts.so"
+ elif os_type == "Darwin": # macOS
+ lib_name = "libhts.dylib"
+ elif os_type == "Windows":
+ lib_name = "hts-3.dll"
+ else:
+ return False # Unsupported OS
+
+ try:
+ ctypes.CDLL(lib_name)
+ return True
+ except Exception:
+ return False
+
+
+def build_htslib(htslib_configure_options, static_mode):
+ current_directory = os.getcwd()
+ os.chdir(os.path.join(current_directory, "htslib"))
+
+ if os.path.exists("config.status"):
+ print("# cyvcf2: config.status exists, skip configure htslib")
+ else:
+ subprocess.run(["autoreconf", "-i"], check=True)
+
+ configure_args = ["./configure"]
+ if static_mode:
+ configure_args.append("CFLAGS=-fPIC")
+ if htslib_configure_options:
+ configure_args.extend(htslib_configure_options.split())
+
+ subprocess.run(configure_args, check=True)
+ subprocess.run(["make"], check=True)
+
+ os.chdir(current_directory)
+
+
+def pre_sdist():
+ current_directory = os.getcwd()
+ os.chdir(os.path.join(current_directory, "htslib"))
+
+ # generate version.h
+ subprocess.run(["make", "htscodecs/htscodecs/version.h"], check=True)
+
+ # remove redundant file
+ redudant_files = ["htscodecs.mk"]
+ for redudant_file in redudant_files:
+ if os.path.exists(redudant_file):
+ os.remove(redudant_file)
+
+ os.chdir(current_directory)
+
+
+class cyvcf2_build_ext(build_ext):
+ def run(self):
+ print("# cyvcf2: htslib mode is {}".format(CYVCF2_HTSLIB_MODE))
+ if CYVCF2_HTSLIB_MODE == "BUILTIN" or not check_libhts():
+ if platform.system() == "Windows":
+ # Windows htslib can't be built internall
+ raise RuntimeError(
+ "Required library 'hts-3.dll' not found. Please install htslib first."
+ )
+
+ print(
+ "# cyvcf2: htslib configure options is {}".format(
+ CYVCF2_HTSLIB_CONFIGURE_OPTIONS
+ )
+ )
+ build_htslib(
+ CYVCF2_HTSLIB_CONFIGURE_OPTIONS, CYVCF2_HTSLIB_MODE == "BUILTIN"
+ )
+
+ if CYVCF2_HTSLIB_MODE == "BUILTIN":
+ # add htslib linked libraries
+ # libcrypto is bundled with libssl, capabale to replace libssl
+ all_dynamic_libs = {
+ "z",
+ "bz2",
+ "lzma",
+ "curl",
+ "deflate",
+ "crypto",
+ }
+
+ extra_libs = []
+ # read the htslib config.status file to get the linked libraries
+ with open(os.path.join("htslib", "config.status"), "r") as f:
+ for line in f:
+ if 'S["static_LIBS"]' in line:
+ linked_libs_str = line.split("=")[1].strip()[1:-1]
+ print("# cyvcf2: htslib librarys is {}".format(linked_libs_str))
+ linked_libs = linked_libs_str.split()
+ for lib in linked_libs:
+ if lib[2:] in all_dynamic_libs: # remove -l prefix
+ extra_libs.append(lib[2:])
+
+ self.extensions[0].libraries = self.extensions[0].libraries + extra_libs
+
+ super().run()
+
+
+class cyvcf2_sdist(sdist):
+ def run(self):
+ if platform.system() == "Windows":
+ raise RuntimeError("cyvcf2 Can't build sdist on Windows")
+
+ pre_sdist()
+
+ super().run()
+
+
+class clean_ext(Command):
+ description = "clean up Cython temporary files and htslib build files"
+ user_options = []
+
+ def initialize_options(self):
+ pass
+
+ def finalize_options(self):
+ pass
+
+ def run(self):
+ print("cleaning build files")
+ if os.path.exists("build"):
+ shutil.rmtree("build")
+
+ if os.path.exists("cyvcf2.egg-info"):
+ shutil.rmtree("cyvcf2.egg-info")
+
+ print("cleaning Cython temporary files")
+ cyvcf2_c_path = os.path.join("cyvcf2", "cyvcf2.c")
+ if os.path.exists(cyvcf2_c_path):
+ os.remove(cyvcf2_c_path)
+
+ lib_files = glob.glob("cyvcf2/cyvcf2.cpython-*")
+ for file in lib_files:
+ os.remove(file)
+
+ if platform.system() != "Windows":
+ print("cleaning htslib build files")
+ current_directory = os.getcwd()
+ os.chdir(os.path.join(current_directory, "htslib"))
+ subprocess.run(["make", "distclean"], check=True)
+ os.chdir(current_directory)
+
+
+# How to link against HTSLIB
+# BUILTIN: build and static link against htslib from
+# builtin htslib code (default)
+# EXTERNAL: use shared libhts.so compiled outside of
+# cyvcf2
+if platform.system() == "Windows":
+ # can't static link to htslib on Windows
+ htslib_mode_default = "EXTERNAL"
+else:
+ htslib_mode_default = "BUILTIN"
+
+CYVCF2_HTSLIB_MODE = os.environ.get("CYVCF2_HTSLIB_MODE", htslib_mode_default)
+
+if platform.system() == "Windows" and CYVCF2_HTSLIB_MODE == "BUILTIN":
+ print(
+ "# cyvcf2 WARNING: The use of cyvcf2 on Windows is experimental. It will not work when statically linked to htslib. Fallback to htslib EXTERNAL mode"
+ )
+ CYVCF2_HTSLIB_MODE = "EXTERNAL"
+
+CYVCF2_HTSLIB_CONFIGURE_OPTIONS = os.environ.get(
+ "CYVCF2_HTSLIB_CONFIGURE_OPTIONS", None
+)
+
+htslib_objects = []
+htslib_librarys = []
+htslib_library_dirs = []
+
+if CYVCF2_HTSLIB_MODE == "BUILTIN":
+ htslib_objects = ["htslib/libhts.a"]
+else:
+ htslib_librarys = ["hts"]
+ if not check_libhts():
+ htslib_library_dirs = ["htslib"]
+
+htslib_include_dirs = ["htslib", "htslib/htslib"]
+
# Build the Cython extension by statically linking to the bundled htslib
-sources = [
- x for x in glob.glob('htslib/*.c')
- if not any(e in x for e in ['irods', 'plugin'])
+sources = ["cyvcf2/cyvcf2.pyx", "cyvcf2/helpers.c"]
+
+extensions = [
+ Extension(
+ "cyvcf2.cyvcf2",
+ sources,
+ extra_objects=htslib_objects,
+ libraries=htslib_librarys,
+ extra_compile_args=[
+ "-Wno-sign-compare",
+ "-Wno-unused-function",
+ "-Wno-strict-prototypes",
+ "-Wno-unused-result",
+ "-Wno-discarded-qualifiers",
+ ],
+ include_dirs=["cyvcf2", np.get_include()] + htslib_include_dirs,
+ library_dirs=htslib_library_dirs,
+ )
]
-sources += glob.glob('htslib/cram/*.c')
-sources += glob.glob('htslib/htscodecs/htscodecs/*.c')
-# Exclude the htslib sources containing main()'s
-sources = [x for x in sources if not x.endswith(('htsfile.c', 'tabix.c', 'bgzip.c'))]
-sources.append('cyvcf2/helpers.c')
-
-extra_libs = []
-if platform.system() != 'Darwin':
- extra_libs.append('crypt')
-if bool(int(os.getenv("LIBDEFLATE", 0))):
- extra_libs.append('deflate')
-
-extensions = [Extension("cyvcf2.cyvcf2",
- ["cyvcf2/cyvcf2.pyx"] + sources,
- libraries=['z', 'bz2', 'lzma', 'curl', 'ssl'] + extra_libs,
- extra_compile_args=["-Wno-sign-compare", "-Wno-unused-function",
- "-Wno-strict-prototypes",
- "-Wno-unused-result", "-Wno-discarded-qualifiers"],
- include_dirs=['htslib', 'cyvcf2', np.get_include()])]
CYTHONIZE = bool(int(os.getenv("CYTHONIZE", 0)))
@@ -94,14 +283,19 @@ setup(
author_email="bpederse at gmail.com",
version=get_version(),
ext_modules=extensions,
- packages=['cyvcf2', 'cyvcf2.tests'],
+ packages=["cyvcf2"],
entry_points=dict(
console_scripts=[
- 'cyvcf2 = cyvcf2.__main__:cli',
+ "cyvcf2 = cyvcf2.__main__:cli",
],
),
python_requires=">=3.7",
- install_requires=['numpy', 'coloredlogs', 'click'],
+ install_requires=["numpy", "coloredlogs", "click"],
include_package_data=True,
zip_safe=False,
+ cmdclass={
+ "clean_ext": clean_ext,
+ "build_ext": cyvcf2_build_ext,
+ "sdist": cyvcf2_sdist,
+ },
)
View it on GitLab: https://salsa.debian.org/med-team/cyvcf2/-/compare/ad4fe3d00ebe7805931e81ff2416b2e5125e4935...f957b105421a30a2ce002d99b91b7201db059df0
--
View it on GitLab: https://salsa.debian.org/med-team/cyvcf2/-/compare/ad4fe3d00ebe7805931e81ff2416b2e5125e4935...f957b105421a30a2ce002d99b91b7201db059df0
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20240207/5b458f55/attachment-0001.htm>
More information about the debian-med-commit
mailing list