[Git][debian-gis-team/netcdf4-python][upstream] New upstream version 1.6.0
Bas Couwenberg (@sebastic)
gitlab at salsa.debian.org
Sat Jun 25 06:19:58 BST 2022
Bas Couwenberg pushed to branch upstream at Debian GIS Project / netcdf4-python
Commits:
9f65992d by Bas Couwenberg at 2022-06-25T06:34:45+02:00
New upstream version 1.6.0
- - - - -
25 changed files:
- .github/workflows/build.yml
- + .github/workflows/build_master.yml
- .github/workflows/miniconda.yml
- Changelog
- MANIFEST.in
- README.md
- docs/index.html
- + examples/bench_compress4.py
- include/mpi-compat.h
- include/netCDF4.pxi
- setup.py
- src/netCDF4/__init__.py
- src/netCDF4/_netCDF4.pyx
- + src/netCDF4/plugins/empty.txt
- src/netCDF4/utils.py
- + test/issue1152.nc
- test/run_all.py
- test/tst_compression.py
- + test/tst_compression_blosc.py
- + test/tst_compression_bzip2.py
- + test/tst_compression_quant.py
- + test/tst_compression_szip.py
- + test/tst_compression_zstd.py
- test/tst_dims.py
- test/tst_masked.py
Changes:
=====================================
.github/workflows/build.yml
=====================================
@@ -6,7 +6,7 @@ jobs:
runs-on: ubuntu-latest
env:
PNETCDF_VERSION: 1.12.1
- NETCDF_VERSION: 4.8.0
+ NETCDF_VERSION: 4.9.0
NETCDF_DIR: ${{ github.workspace }}/..
NETCDF_EXTRA_CONFIG: --enable-pnetcdf
CC: mpicc.mpich
@@ -26,7 +26,7 @@ jobs:
- name: Install Ubuntu Dependencies
run: |
sudo apt-get update
- sudo apt-get install mpich libmpich-dev libhdf5-mpich-dev libcurl4-openssl-dev
+ sudo apt-get install mpich libmpich-dev libhdf5-mpich-dev libcurl4-openssl-dev bzip2 libsnappy-dev libblosc-dev libzstd-dev
echo "Download and build PnetCDF version ${PNETCDF_VERSION}"
wget https://parallel-netcdf.github.io/Release/pnetcdf-${PNETCDF_VERSION}.tar.gz
tar -xzf pnetcdf-${PNETCDF_VERSION}.tar.gz
@@ -36,7 +36,7 @@ jobs:
make install
popd
echo "Download and build netCDF version ${NETCDF_VERSION}"
- wget ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-c-${NETCDF_VERSION}.tar.gz
+ wget https://downloads.unidata.ucar.edu/netcdf-c/4.9.0/netcdf-c-${NETCDF_VERSION}.tar.gz
tar -xzf netcdf-c-${NETCDF_VERSION}.tar.gz
pushd netcdf-c-${NETCDF_VERSION}
export CPPFLAGS="-I/usr/include/hdf5/mpich -I${NETCDF_DIR}/include"
@@ -61,6 +61,7 @@ jobs:
- name: Install netcdf4-python
run: |
export PATH=${NETCDF_DIR}/bin:${PATH}
+ export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c-${NETCDF_VERSION}/plugins/plugindir
python setup.py install
- name: Test
run: |
@@ -97,6 +98,7 @@ jobs:
run: |
export PATH=${NETCDF_DIR}/bin:${PATH}
python setup.py --version
- pip wheel . -w dist --no-deps
+ check-manifest --version
check-manifest --verbose
+ pip wheel . -w dist --no-deps
twine check dist/*
=====================================
.github/workflows/build_master.yml
=====================================
@@ -0,0 +1,78 @@
+name: Build and Test on Linux with netcdf-c github master
+on: [push, pull_request]
+jobs:
+ build-linux:
+ name: Python (${{ matrix.python-version }})
+ runs-on: ubuntu-latest
+ env:
+ NETCDF_DIR: ${{ github.workspace }}/..
+ CC: mpicc.mpich
+ #NO_NET: 1
+ strategy:
+ matrix:
+ python-version: ["3.9"]
+ steps:
+
+ - uses: actions/checkout at v2
+
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python at v2
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Install Ubuntu Dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install mpich libmpich-dev libhdf5-mpich-dev libcurl4-openssl-dev bzip2 libsnappy-dev libblosc-dev libzstd-dev
+ echo "Download and build netCDF github master"
+ git clone https://github.com/Unidata/netcdf-c
+ pushd netcdf-c
+ export CPPFLAGS="-I/usr/include/hdf5/mpich -I${NETCDF_DIR}/include"
+ export LDFLAGS="-L${NETCDF_DIR}/lib"
+ export LIBS="-lhdf5_mpich_hl -lhdf5_mpich -lm -lz"
+ autoreconf -i
+ ./configure --prefix $NETCDF_DIR --enable-netcdf-4 --enable-shared --enable-dap --enable-parallel4
+ make -j 2
+ make install
+ popd
+
+# - name: The job has failed
+# if: ${{ failure() }}
+# run: |
+# cd netcdf-c-${NETCDF_VERSION}
+# cat config.log
+
+ - name: Install python dependencies via pip
+ run: |
+ python -m pip install --upgrade pip
+ pip install numpy cython cftime pytest twine wheel check-manifest mpi4py
+
+ - name: Install netcdf4-python
+ run: |
+ export PATH=${NETCDF_DIR}/bin:${PATH}
+ export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c/plugins/plugindir
+ python setup.py install
+ - name: Test
+ run: |
+ export PATH=${NETCDF_DIR}/bin:${PATH}
+ #export HDF5_PLUGIN_PATH=${NETCDF_DIR}/plugins/plugindir
+ python checkversion.py
+ # serial
+ cd test
+ python run_all.py
+ # parallel
+ cd ../examples
+ mpirun.mpich -np 4 python mpi_example.py
+ if [ $? -ne 0 ] ; then
+ echo "hdf5 mpi test failed!"
+ exit 1
+ else
+ echo "hdf5 mpi test passed!"
+ fi
+ mpirun.mpich -np 4 python mpi_example_compressed.py
+ if [ $? -ne 0 ] ; then
+ echo "hdf5 compressed mpi test failed!"
+ exit 1
+ else
+ echo "hdf5 compressed mpi test passed!"
+ fi
=====================================
.github/workflows/miniconda.yml
=====================================
@@ -78,6 +78,7 @@ jobs:
export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH"
which mpirun
mpirun --version
+ #mpirun -np 4 --oversubscribe python mpi_example.py # for openmpi
mpirun -np 4 python mpi_example.py
if [ $? -ne 0 ] ; then
echo "hdf5 mpi test failed!"
=====================================
Changelog
=====================================
@@ -1,3 +1,34 @@
+ version 1.6.0 (tag v1.6.0rel)
+==============================
+ * add support for new quantization functionality in netcdf-c 4.9.0 via "signficant_digits"
+ and "quantize_mode" kwargs in Dataset.createVariable. Default quantization_mode is "BitGroom",
+ but alternate methods "BitRound" and GranularBitRound" also supported.
+ * opening a Dataset in append mode (mode = 'a' or 'r+') creates a Dataset
+ if one does not already exist (similar to python open builtin). Issue #1144.
+ Added a mode='x' option (as in python open) which is the same as mode='w' with
+ clobber=False.
+ * allow createVariable to accept either Dimension instances or Dimension
+ names in "dimensions" tuple kwarg (issue #1145).
+ * remove all vestiges of python 2 in _netCDF4.pyx and set cython language_level
+ directive to 3 in setup.py.
+ * add 'compression' kwarg to createVariable to enable new compression
+ functionality in netcdf-c 4.9.0. 'None','zlib','szip','zstd','bzip2'
+ 'blosc_lz','blosc_lz4','blosc_lz4hc','blosc_zlib' and 'blosc_zstd'
+ are currently supported. 'blosc_shuffle',
+ 'szip_mask' and 'szip_pixels_per_block' kwargs also added.
+ compression='zlib' is equivalent to (the now deprecated) zlib=True.
+ If the environment variable NETCDF_PLUGIN_DIR is set to point to the
+ directory with the compression plugin lib__nc* files, then the compression plugins will
+ be installed within the package and be automatically available (the binary
+ wheels have this). Otherwise, the environment variable HDF5_PLUGIN_PATH
+ needs to be set at runtime to point to plugins in order to use the new compression
+ options.
+ * MFDataset did not aggregate 'name' variable attribute (issue #1153).
+ * issue warning instead of raising an exception if missing_value or
+ _FillValue can't be cast to the variable type when creating a
+ masked array (issue #1152).
+ * Define MPI_Session for compatibility with current mpi4py (PR #1156).
+
version 1.5.8 (tag v1.5.8rel)
==============================
* Fix Enum bug (issue #1128): the enum_dict member of an EnumType read from a file
=====================================
MANIFEST.in
=====================================
@@ -6,11 +6,14 @@ include Changelog
include setup.cfg
include examples/*py
include examples/README.md
+exclude examples/data
include test/*py
include test/*nc
include src/netCDF4/__init__.py
include src/netCDF4/_netCDF4.pyx
+exclude src/netCDF4/_netCDF4.c
include src/netCDF4/utils.py
+include src/netCDF4/plugins/empty.txt
include include/netCDF4.pxi
include include/mpi-compat.h
include include/membuf.pyx
=====================================
README.md
=====================================
@@ -10,6 +10,13 @@
## News
For details on the latest updates, see the [Changelog](https://github.com/Unidata/netcdf4-python/blob/master/Changelog).
+??/??/2022: Version [1.6.0](https://pypi.python.org/pypi/netCDF4/1.6.0) released. Support
+for quantization (bit-grooming and bit-rounding) functionality in netcdf-c 4.9.0 which can
+dramatically improve compression. Dataset.createVariable now accepts dimension instances (instead
+of just dimension names). 'compression' kwarg added to Dataset.createVariable to support szip as
+well as new compression algorithms available in netcdf-c 4.9.0 through compression plugins (such
+as zstd, bzip2 and blosc). Working arm64 wheels for Apple M1 Silicon now available on pypi.
+
10/31/2021: Version [1.5.8](https://pypi.python.org/pypi/netCDF4/1.5.8) released. Fix Enum bug, add binary wheels for aarch64 and python 3.10.
6/22/2021: Version [1.5.7](https://pypi.python.org/pypi/netCDF4/1.5.7) released.
=====================================
docs/index.html
=====================================
The diff for this file was not included because it is too large.
=====================================
examples/bench_compress4.py
=====================================
@@ -0,0 +1,53 @@
+from __future__ import print_function
+# benchmark reads and writes, with and without compression.
+# tests all four supported file formats.
+from numpy.random.mtrand import uniform
+import netCDF4
+from timeit import Timer
+import os, sys
+
+# use real data.
+URL="http://www.esrl.noaa.gov/psd/thredds/dodsC/Datasets/ncep.reanalysis/pressure/hgt.1990.nc"
+nc = netCDF4.Dataset(URL)
+
+# use real 500 hPa geopotential height data.
+n1dim = 100
+n3dim = 73
+n4dim = 144
+ntrials = 10
+sys.stdout.write('reading and writing a %s by %s by %s random array ..\n'%(n1dim,n3dim,n4dim))
+sys.stdout.write('(average of %s trials)\n\n' % ntrials)
+array = nc.variables['hgt'][0:n1dim,5,:,:]
+
+
+def write_netcdf(filename,nsd,quantize_mode='BitGroom'):
+ file = netCDF4.Dataset(filename,'w',format='NETCDF4')
+ file.createDimension('n1', None)
+ file.createDimension('n3', n3dim)
+ file.createDimension('n4', n4dim)
+ foo = file.createVariable('data',\
+ 'f4',('n1','n3','n4'),\
+ zlib=True,shuffle=True,\
+ quantize_mode=quantize_mode,\
+ significant_digits=nsd)
+ foo[:] = array
+ file.close()
+
+def read_netcdf(filename):
+ file = netCDF4.Dataset(filename)
+ data = file.variables['data'][:]
+ file.close()
+
+for sigdigits in range(1,5,1):
+ sys.stdout.write('testing compression with significant_digits=%s...\n' %\
+ sigdigits)
+ write_netcdf('test.nc',sigdigits)
+ read_netcdf('test.nc')
+ # print out size of resulting files with standard quantization.
+ sys.stdout.write('size of test.nc = %s\n'%repr(os.stat('test.nc').st_size))
+ sys.stdout.write("testing compression with significant_digits=%s and 'GranularBitRound'...\n" %\
+ sigdigits)
+ write_netcdf('test.nc',sigdigits,quantize_mode='GranularBitRound')
+ read_netcdf('test.nc')
+ # print out size of resulting files with alternate quantization.
+ sys.stdout.write('size of test.nc = %s\n'%repr(os.stat('test.nc').st_size))
=====================================
include/mpi-compat.h
=====================================
@@ -11,4 +11,9 @@ typedef void *PyMPI_MPI_Message;
#define MPI_Message PyMPI_MPI_Message
#endif
+#if (MPI_VERSION < 4) && !defined(PyMPI_HAVE_MPI_Session)
+typedef void *PyMPI_MPI_Session;
+#define MPI_Session PyMPI_MPI_Session
+#endif
+
#endif/*MPI_COMPAT_H*/
=====================================
include/netCDF4.pxi
=====================================
@@ -216,8 +216,6 @@ cdef extern from "netcdf.h":
NC_ENDIAN_NATIVE
NC_ENDIAN_LITTLE
NC_ENDIAN_BIG
- NC_SZIP_EC_OPTION_MASK # entropy encoding
- NC_SZIP_NN_OPTION_MASK # nearest neighbor encoding
const_char_ptr *nc_inq_libvers() nogil
const_char_ptr *nc_strerror(int ncerr)
int nc_create(char *path, int cmode, int *ncidp)
@@ -691,6 +689,43 @@ cdef extern from "netcdf.h":
int nc_inq_enum_ident(int ncid, nc_type xtype, long long value, char *identifier) nogil
+IF HAS_QUANTIZATION_SUPPORT:
+ cdef extern from "netcdf.h":
+ cdef enum:
+ NC_NOQUANTIZE
+ NC_QUANTIZE_BITGROOM
+ NC_QUANTIZE_GRANULARBR
+ NC_QUANTIZE_BITROUND
+ int nc_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd)
+ int nc_inq_var_quantize(int ncid, int varid, int *quantize_modep, int *nsdp) nogil
+
+IF HAS_SZIP_SUPPORT:
+ cdef extern from "netcdf.h":
+ int nc_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd)
+ int nc_inq_var_quantize(int ncid, int varid, int *quantize_modep, int *nsdp) nogil
+ int nc_def_var_szip(int ncid, int varid, int options_mask, int pixels_per_bloc)
+ int nc_inq_var_szip(int ncid, int varid, int *options_maskp, int *pixels_per_blockp)
+
+IF HAS_ZSTANDARD_SUPPORT:
+ cdef extern from "netcdf_filter.h":
+ cdef enum:
+ H5Z_FILTER_ZSTANDARD
+ int nc_def_var_zstandard(int ncid, int varid, int level)
+ int nc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *levelp)
+ int nc_inq_filter_avail(int ncid, unsigned id)
+
+IF HAS_BZIP2_SUPPORT:
+ cdef extern from "netcdf_filter.h":
+ cdef enum:
+ H5Z_FILTER_BZIP2
+ int nc_def_var_bzip2(int ncid, int varid, int level)
+ int nc_inq_var_bzip2(int ncid, int varid, int* hasfilterp, int *levelp)
+
+IF HAS_BLOSC_SUPPORT:
+ cdef extern from "netcdf_filter.h":
+ int nc_def_var_blosc(int ncid, int varid, unsigned subcompressor, unsigned level, unsigned blocksize, unsigned addshuffle)
+ int nc_inq_var_blosc(int ncid, int varid, int* hasfilterp, unsigned* subcompressorp, unsigned* levelp, unsigned* blocksizep, unsigned* addshufflep)
+
IF HAS_NC_OPEN_MEM:
cdef extern from "netcdf_mem.h":
int nc_open_mem(const char *path, int mode, size_t size, void* memory, int *ncidp)
=====================================
setup.py
=====================================
@@ -1,7 +1,8 @@
-import os, sys, subprocess
+import os, sys, subprocess, glob
import os.path as osp
+import shutil
import configparser
-from setuptools import setup, Extension
+from setuptools import setup, Extension, find_namespace_packages
from distutils.dist import Distribution
setuptools_extra_kwargs = {
@@ -65,6 +66,11 @@ def check_api(inc_dirs,netcdf_lib_version):
has_parallel_support = False
has_parallel4_support = False
has_pnetcdf_support = False
+ has_szip_support = False
+ has_quantize = False
+ has_zstandard = False
+ has_bzip2 = False
+ has_blosc = False
for d in inc_dirs:
try:
@@ -73,6 +79,7 @@ def check_api(inc_dirs,netcdf_lib_version):
continue
has_nc_open_mem = os.path.exists(os.path.join(d, 'netcdf_mem.h'))
+ has_nc_filter = os.path.exists(os.path.join(d, 'netcdf_filter.h'))
for line in f:
if line.startswith('nc_rename_grp'):
@@ -83,6 +90,8 @@ def check_api(inc_dirs,netcdf_lib_version):
has_nc_inq_format_extended = True
if line.startswith('#define NC_FORMAT_64BIT_DATA'):
has_cdf5_format = True
+ if line.startswith('nc_def_var_quantize'):
+ has_quantize = True
if has_nc_open_mem:
try:
@@ -93,6 +102,19 @@ def check_api(inc_dirs,netcdf_lib_version):
if line.startswith('EXTERNL int nc_create_mem'):
has_nc_create_mem = True
+ if has_nc_filter:
+ try:
+ f = open(os.path.join(d, 'netcdf_filter.h'), **open_kwargs)
+ except IOError:
+ continue
+ for line in f:
+ if line.startswith('EXTERNL int nc_def_var_zstandard'):
+ has_zstandard = True
+ if line.startswith('EXTERNL int nc_def_var_bzip2'):
+ has_bzip2 = True
+ if line.startswith('EXTERNL int nc_def_var_blosc'):
+ has_blosc = True
+
ncmetapath = os.path.join(d,'netcdf_meta.h')
if os.path.exists(ncmetapath):
for line in open(ncmetapath):
@@ -104,6 +126,8 @@ def check_api(inc_dirs,netcdf_lib_version):
has_parallel4_support = bool(int(line.split()[2]))
if line.startswith('#define NC_HAS_PNETCDF'):
has_pnetcdf_support = bool(int(line.split()[2]))
+ if line.startswith('#define NC_HAS_SZIP_WRITE'):
+ has_szip_support = bool(int(line.split()[2]))
# NC_HAS_PARALLEL4 missing in 4.6.1 (issue #964)
if not has_parallel4_support and has_parallel_support and not has_pnetcdf_support:
has_parallel4_support = True
@@ -116,7 +140,8 @@ def check_api(inc_dirs,netcdf_lib_version):
return has_rename_grp, has_nc_inq_path, has_nc_inq_format_extended, \
has_cdf5_format, has_nc_open_mem, has_nc_create_mem, \
- has_parallel4_support, has_pnetcdf_support
+ has_parallel4_support, has_pnetcdf_support, has_szip_support, has_quantize, \
+ has_zstandard, has_bzip2, has_blosc
def getnetcdfvers(libdirs):
@@ -382,7 +407,7 @@ if os.environ.get("CONDA_PREFIX"):
dirstosearch.append(os.environ["CONDA_PREFIX"]) # linux,macosx
dirstosearch.append(os.path.join(os.environ["CONDA_PREFIX"],'Library')) # windows
dirstosearch += [os.path.expanduser('~'), '/usr/local', '/sw', '/opt',
- '/opt/local', '/usr']
+ '/opt/local', '/opt/homebrew', '/usr']
# try nc-config first
if USE_NCCONFIG and HAS_NCCONFIG: # Try nc-config.
@@ -529,7 +554,8 @@ if 'sdist' not in sys.argv[1:] and 'clean' not in sys.argv[1:] and '--version' n
# this determines whether renameGroup and filepath methods will work.
has_rename_grp, has_nc_inq_path, has_nc_inq_format_extended, \
has_cdf5_format, has_nc_open_mem, has_nc_create_mem, \
- has_parallel4_support, has_pnetcdf_support = \
+ has_parallel4_support, has_pnetcdf_support, has_szip_support, has_quantize, \
+ has_zstandard, has_bzip2, has_blosc = \
check_api(inc_dirs,netcdf_lib_version)
# for netcdf 4.4.x CDF5 format is always enabled.
if netcdf_lib_version is not None and\
@@ -601,6 +627,41 @@ if 'sdist' not in sys.argv[1:] and 'clean' not in sys.argv[1:] and '--version' n
sys.stdout.write('netcdf lib does not have pnetcdf parallel functions\n')
f.write('DEF HAS_PNETCDF_SUPPORT = 0\n')
+ if has_quantize:
+ sys.stdout.write('netcdf lib has bit-grooming/quantization functions\n')
+ f.write('DEF HAS_QUANTIZATION_SUPPORT = 1\n')
+ else:
+ sys.stdout.write('netcdf lib does not have bit-grooming/quantization functions\n')
+ f.write('DEF HAS_QUANTIZATION_SUPPORT = 0\n')
+
+ if has_zstandard:
+ sys.stdout.write('netcdf lib has zstandard compression functions\n')
+ f.write('DEF HAS_ZSTANDARD_SUPPORT = 1\n')
+ else:
+ sys.stdout.write('netcdf lib does not have zstandard compression functions\n')
+ f.write('DEF HAS_ZSTANDARD_SUPPORT = 0\n')
+
+ if has_bzip2:
+ sys.stdout.write('netcdf lib has bzip2 compression functions\n')
+ f.write('DEF HAS_BZIP2_SUPPORT = 1\n')
+ else:
+ sys.stdout.write('netcdf lib does not have bzip2 compression functions\n')
+ f.write('DEF HAS_BZIP2_SUPPORT = 0\n')
+
+ if has_blosc:
+ sys.stdout.write('netcdf lib has blosc compression functions\n')
+ f.write('DEF HAS_BLOSC_SUPPORT = 1\n')
+ else:
+ sys.stdout.write('netcdf lib does not have blosc compression functions\n')
+ f.write('DEF HAS_BLOSC_SUPPORT = 0\n')
+
+ if has_szip_support:
+ sys.stdout.write('netcdf lib has szip compression functions\n')
+ f.write('DEF HAS_SZIP_SUPPORT = 1\n')
+ else:
+ sys.stdout.write('netcdf lib does not have szip compression functions\n')
+ f.write('DEF HAS_SZIP_SUPPORT = 0\n')
+
f.close()
if has_parallel4_support or has_pnetcdf_support:
@@ -616,9 +677,34 @@ if 'sdist' not in sys.argv[1:] and 'clean' not in sys.argv[1:] and '--version' n
library_dirs=lib_dirs,
include_dirs=inc_dirs + ['include'],
runtime_library_dirs=runtime_lib_dirs)]
+ # set language_level directive to 3
+ for e in ext_modules:
+ e.cython_directives = {'language_level': "3"} #
else:
ext_modules = None
+# if NETCDF_PLUGIN_DIR set, install netcdf-c compression plugins inside package
+# (should point to location of lib__nc* files built by netcdf-c)
+copied_plugins=False
+if os.environ.get("NETCDF_PLUGIN_DIR"):
+ plugin_dir = os.environ.get("NETCDF_PLUGIN_DIR")
+ plugins = glob.glob(os.path.join(plugin_dir, "lib__nc*"))
+ if not plugins:
+ sys.stdout.write('no plugin files in NETCDF_PLUGIN_DIR, not installing..\n')
+ data_files = []
+ else:
+ data_files = plugins
+ sys.stdout.write('installing netcdf compression plugins from %s ...\n' % plugin_dir)
+ sofiles = [os.path.basename(sofilepath) for sofilepath in data_files]
+ sys.stdout.write(repr(sofiles)+'\n')
+ if 'sdist' not in sys.argv[1:] and 'clean' not in sys.argv[1:] and '--version' not in sys.argv[1:]:
+ for f in data_files:
+ shutil.copy(f, osp.join(os.getcwd(),osp.join(osp.join('src','netCDF4'),'plugins')))
+ copied_plugins=True
+else:
+ sys.stdout.write('NETCDF_PLUGIN_DIR not set, no netcdf compression plugins installed\n')
+ data_files = []
+
setup(name="netCDF4",
cmdclass=cmdclass,
version=extract_version(netcdf4_src_pyx),
@@ -642,7 +728,15 @@ setup(name="netCDF4",
"Topic :: Software Development :: Libraries :: Python Modules",
"Topic :: System :: Archiving :: Compression",
"Operating System :: OS Independent"],
- packages=['netCDF4'],
+ packages=find_namespace_packages(where="src"),
package_dir={'':'src'},
+ package_data={"netCDF4.plugins": ["lib__nc*"]},
ext_modules=ext_modules,
**setuptools_extra_kwargs)
+
+# remove plugin files copied from outside source tree
+if copied_plugins:
+ for f in sofiles:
+ filepath = osp.join(osp.join(osp.join('src','netCDF4'),'plugins'),f)
+ if os.path.exists(filepath):
+ os.remove(filepath)
=====================================
src/netCDF4/__init__.py
=====================================
@@ -7,6 +7,15 @@ from ._netCDF4 import (__version__, __netcdf4libversion__, __hdf5libversion__,
__has_rename_grp__, __has_nc_inq_path__,
__has_nc_inq_format_extended__, __has_nc_open_mem__,
__has_nc_create_mem__, __has_cdf5_format__,
- __has_parallel4_support__, __has_pnetcdf_support__)
+ __has_parallel4_support__, __has_pnetcdf_support__,
+ __has_quantization_support__, __has_zstandard_support__,
+ __has_bzip2_support__, __has_blosc_support__, __has_szip_support__)
+import os
__all__ =\
['Dataset','Variable','Dimension','Group','MFDataset','MFTime','CompoundType','VLType','date2num','num2date','date2index','stringtochar','chartostring','stringtoarr','getlibversion','EnumType','get_chunk_cache','set_chunk_cache']
+# if HDF5_PLUGIN_PATH not set, point to package path if plugins live there
+pluginpath = os.path.join(__path__[0],'plugins')
+if 'HDF5_PLUGIN_PATH' not in os.environ and\
+ (os.path.exists(os.path.join(pluginpath,'lib__nczhdf5filters.so')) or\
+ os.path.exists(os.path.join(pluginpath,'lib__nczhdf5filters.dylib'))):
+ os.environ['HDF5_PLUGIN_PATH']=pluginpath
=====================================
src/netCDF4/_netCDF4.pyx
=====================================
@@ -1,5 +1,5 @@
"""
-Version 1.5.8
+Version 1.6.0
-------------
# Introduction
@@ -15,7 +15,7 @@ files that are readable by HDF5 clients. The API modelled after
and should be familiar to users of that module.
Most new features of netCDF 4 are implemented, such as multiple
-unlimited dimensions, groups and zlib data compression. All the new
+unlimited dimensions, groups and data compression. All the new
numeric data types (such as 64 bit and unsigned integer types) are
implemented. Compound (struct), variable length (vlen) and
enumerated (enum) data types are supported, but not the opaque data type.
@@ -58,6 +58,12 @@ types) are not supported.
If the dependencies are not found
in any of the paths specified by environment variables, then standard locations
(such as `/usr` and `/usr/local`) are searched.
+ - if the env var `NETCDF_PLUGIN_DIR` is set to point to the location of the netcdf-c compression
+ plugins built by netcdf >= 4.9.0, they will be installed inside the package. In this
+ case `HDF5_PLUGIN_PATH` will be set to the package installation path on import,
+ so the extra compression algorithms available in netcdf-c >= 4.9.0 will automatically
+ be available. Otherwise, the user will have to set `HDF5_PLUGIN_PATH` explicitly
+ to have access to the extra compression plugins.
- run `python setup.py build`, then `python setup.py install` (as root if
necessary).
- run the tests in the 'test' directory by running `python run_all.py`.
@@ -641,17 +647,19 @@ datasets.
## Efficient compression of netCDF variables
-Data stored in netCDF 4 `Variable` objects can be compressed and
-decompressed on the fly. The parameters for the compression are
-determined by the `zlib`, `complevel` and `shuffle` keyword arguments
-to the `Dataset.createVariable` method. To turn on
-compression, set `zlib=True`. The `complevel` keyword regulates the
-speed and efficiency of the compression (1 being fastest, but lowest
+Data stored in netCDF `Variable` objects can be compressed and
+decompressed on the fly. The compression algorithm used is determined
+by the `compression` keyword argument to the `Dataset.createVariable` method.
+`zlib` compression is always available, `szip` is available if the linked HDF5
+library supports it, and `zstd`, `bzip2`, `blosc_lz`,`blosc_lz4`,`blosc_lz4hc`,
+`blosc_zlib` and `blosc_zstd` are available via optional external plugins.
+The `complevel` keyword regulates the
+speed and efficiency of the compression for `zlib`, `bzip` and `zstd` (1 being fastest, but lowest
compression ratio, 9 being slowest but best compression ratio). The
default value of `complevel` is 4. Setting `shuffle=False` will turn
off the HDF5 shuffle filter, which de-interlaces a block of data before
-compression by reordering the bytes. The shuffle filter can
-significantly improve compression ratios, and is on by default. Setting
+`zlib` compression by reordering the bytes. The shuffle filter can
+significantly improve compression ratios, and is on by default if `compression=zlib`. Setting
`fletcher32` keyword argument to
`Dataset.createVariable` to `True` (it's `False` by
default) enables the Fletcher32 checksum algorithm for error detection.
@@ -662,18 +670,30 @@ and `endian` keyword arguments to
are relevant for `NETCDF4` and `NETCDF4_CLASSIC` files (where the
underlying file format is HDF5) and are silently ignored if the file
format is `NETCDF3_CLASSIC`, `NETCDF3_64BIT_OFFSET` or `NETCDF3_64BIT_DATA`.
+If the HDF5 library is built with szip support, compression=`szip` can also
+be used (in conjunction with the `szip_coding` and `szip_pixels_per_block` keyword
+arguments).
If your data only has a certain number of digits of precision (say for
example, it is temperature data that was measured with a precision of
-0.1 degrees), you can dramatically improve zlib compression by
-quantizing (or truncating) the data using the `least_significant_digit`
-keyword argument to `Dataset.createVariable`. The least
-significant digit is the power of ten of the smallest decimal place in
+0.1 degrees), you can dramatically improve compression by
+quantizing (or truncating) the data. There are two methods supplied for
+doing this. You can use the `least_significant_digit`
+keyword argument to `Dataset.createVariable` to specify
+the power of ten of the smallest decimal place in
the data that is a reliable value. For example if the data has a
precision of 0.1, then setting `least_significant_digit=1` will cause
data the data to be quantized using `numpy.around(scale*data)/scale`, where
scale = 2**bits, and bits is determined so that a precision of 0.1 is
-retained (in this case bits=4). Effectively, this makes the compression
+retained (in this case bits=4). This is done at the python level and is
+not a part of the underlying C library. Starting with netcdf-c version 4.9.0,
+a quantization capability is provided in the library. This can be
+used via the `significant_digits` `Dataset.createVariable` kwarg (new in
+version 1.6.0).
+The interpretation of `significant_digits` is different than `least_signficant_digit`
+in that it specifies the absolute number of significant digits independent
+of the magnitude of the variable (the floating point exponent).
+Either of these approaches makes the compression
'lossy' instead of 'lossless', that is some precision in the data is
sacrificed for the sake of disk space.
@@ -686,13 +706,19 @@ In our example, try replacing the line
with
```python
->>> temp = rootgrp.createVariable("temp","f4",("time","level","lat","lon",),zlib=True)
+>>> temp = rootgrp.createVariable("temp","f4",("time","level","lat","lon",),compression='zlib')
```
and then
```python
->>> temp = rootgrp.createVariable("temp","f4",("time","level","lat","lon",),zlib=True,least_significant_digit=3)
+>>> temp = rootgrp.createVariable("temp","f4",("time","level","lat","lon",),compression='zlib',least_significant_digit=3)
+```
+
+or with netcdf-c >= 4.9.0
+
+```python
+>>> temp = rootgrp.createVariable("temp","f4",("time","level","lat","lon",),compression='zlib',significant_digits=4)
```
and see how much smaller the resulting files are.
@@ -1204,7 +1230,7 @@ if sys.version_info[0:2] < (3, 7):
# Python 3.7+ guarantees order; older versions need OrderedDict
from collections import OrderedDict
-__version__ = "1.5.8"
+__version__ = "1.6.0"
# Initialize numpy
import posixpath
@@ -1214,6 +1240,7 @@ import weakref
import warnings
import subprocess
import pathlib
+import os
from glob import glob
from numpy import ma
from libc.string cimport memcpy, memset
@@ -1304,6 +1331,11 @@ __has_nc_open_mem__ = HAS_NC_OPEN_MEM
__has_nc_create_mem__ = HAS_NC_CREATE_MEM
__has_parallel4_support__ = HAS_PARALLEL4_SUPPORT
__has_pnetcdf_support__ = HAS_PNETCDF_SUPPORT
+__has_quantization_support__ = HAS_QUANTIZATION_SUPPORT
+__has_zstandard_support__ = HAS_ZSTANDARD_SUPPORT
+__has_bzip2_support__ = HAS_BZIP2_SUPPORT
+__has_blosc_support__ = HAS_BLOSC_SUPPORT
+__has_szip_support__ = HAS_SZIP_SUPPORT
_needsworkaround_issue485 = __netcdf4libversion__ < "4.4.0" or \
(__netcdf4libversion__.startswith("4.4.0") and \
"-development" in __netcdf4libversion__)
@@ -1349,6 +1381,11 @@ _format_dict = {'NETCDF3_CLASSIC' : NC_FORMAT_CLASSIC,
_cmode_dict = {'NETCDF3_CLASSIC' : NC_CLASSIC_MODEL,
'NETCDF4_CLASSIC' : NC_CLASSIC_MODEL | NC_NETCDF4,
'NETCDF4' : NC_NETCDF4}
+# dicts for blosc, szip compressors.
+_blosc_dict={'blosc_lz':0,'blosc_lz4':1,'blosc_lz4hc':2,'blosc_snappy':3,'blosc_zlib':4,'blosc_zstd':5}
+_blosc_dict_inv = {v: k for k, v in _blosc_dict.items()}
+_szip_dict = {'ec': 4, 'nn': 32}
+_szip_dict_inv = {v: k for k, v in _szip_dict.items()}
IF HAS_CDF5_FORMAT:
# NETCDF3_64BIT deprecated, saved for compatibility.
# use NETCDF3_64BIT_OFFSET instead.
@@ -1908,6 +1945,7 @@ cdef _get_vars(group):
grp = grp.parent
free(dimids)
# create new variable instance.
+ dimensions = tuple(_find_dim(group,d) for d in dimensions)
if endianness == '>':
variables[name] = Variable(group, name, datatype, dimensions, id=varid, endian='big')
elif endianness == '<':
@@ -2031,8 +2069,10 @@ strings.
**`mode`**: access mode. `r` means read-only; no data can be
modified. `w` means write; a new file is created, an existing file with
- the same name is deleted. `a` and `r+` mean append (in analogy with
- serial files); an existing file is opened for reading and writing.
+ the same name is deleted. `x` means write, but fail if an existing
+ file with the same name already exists. `a` and `r+` mean append;
+ an existing file is opened for reading and writing, if
+ file does not exist already, one is created.
Appending `s` to modes `r`, `w`, `r+` or `a` will enable unbuffered shared
access to `NETCDF3_CLASSIC`, `NETCDF3_64BIT_OFFSET` or
`NETCDF3_64BIT_DATA` formatted files.
@@ -2044,6 +2084,7 @@ strings.
**`clobber`**: if `True` (default), opening a file with `mode='w'`
will clobber an existing file with the same name. if `False`, an
exception will be raised if a file with the same name already exists.
+ mode=`x` is identical to mode=`w` with clobber=False.
**`format`**: underlying file format (one of `'NETCDF4',
'NETCDF4_CLASSIC', 'NETCDF3_CLASSIC'`, `'NETCDF3_64BIT_OFFSET'` or
@@ -2086,14 +2127,14 @@ strings.
rendered unusable when the parent Dataset instance is garbage collected.
**`memory`**: if not `None`, create or open an in-memory Dataset.
- If mode = 'r', the memory kwarg must contain a memory buffer object
+ If mode = `r`, the memory kwarg must contain a memory buffer object
(an object that supports the python buffer interface).
The Dataset will then be created with contents taken from this block of memory.
- If mode = 'w', the memory kwarg should contain the anticipated size
+ If mode = `w`, the memory kwarg should contain the anticipated size
of the Dataset in bytes (used only for NETCDF3 files). A memory
buffer containing a copy of the Dataset is returned by the
- `Dataset.close` method. Requires netcdf-c version 4.4.1 for mode='r,
- netcdf-c 4.6.2 for mode='w'. To persist the file to disk, the raw
+ `Dataset.close` method. Requires netcdf-c version 4.4.1 for mode=`r`
+ netcdf-c 4.6.2 for mode=`w`. To persist the file to disk, the raw
bytes from the returned buffer can be written into a binary file.
The Dataset can also be re-opened using this memory buffer.
@@ -2164,7 +2205,12 @@ strings.
cmode = NC_MPIIO | _cmode_dict[format]
self._inmemory = False
- if mode == 'w':
+
+ # mode='x' is the same as mode='w' with clobber=False
+ if mode == 'x':
+ mode = 'w'; clobber = False
+
+ if mode == 'w' or (mode in ['a','r+'] and not os.path.exists(filename)):
_set_default_format(format=format)
if memory is not None:
# if memory is not None and mode='w', memory
@@ -2246,7 +2292,7 @@ strings.
ierr = nc_open(path, NC_NOWRITE | NC_SHARE, &grpid)
else:
ierr = nc_open(path, NC_NOWRITE, &grpid)
- elif mode == 'r+' or mode == 'a':
+ elif mode in ['a','r+'] and os.path.exists(filename):
if parallel:
IF HAS_PARALLEL4_SUPPORT or HAS_PNETCDF_SUPPORT:
ierr = nc_open_par(path, NC_WRITE | NC_MPIIO, \
@@ -2257,7 +2303,7 @@ strings.
ierr = nc_open(path, NC_WRITE | NC_DISKLESS, &grpid)
else:
ierr = nc_open(path, NC_WRITE, &grpid)
- elif mode == 'as' or mode == 'r+s':
+ elif mode in ['as','r+s'] and os.path.exists(filename):
if parallel:
# NC_SHARE ignored
IF HAS_PARALLEL4_SUPPORT or HAS_PNETCDF_SUPPORT:
@@ -2269,7 +2315,7 @@ strings.
ierr = nc_open(path, NC_SHARE | NC_DISKLESS, &grpid)
else:
ierr = nc_open(path, NC_SHARE, &grpid)
- elif mode == 'ws':
+ elif mode == 'ws' or (mode in ['as','r+s'] and not os.path.exists(filename)):
_set_default_format(format=format)
if clobber:
if parallel:
@@ -2302,7 +2348,7 @@ strings.
else:
ierr = nc_create(path, NC_SHARE | NC_NOCLOBBER, &grpid)
else:
- raise ValueError("mode must be 'w', 'r', 'a' or 'r+', got '%s'" % mode)
+ raise ValueError("mode must be 'w', 'x', 'r', 'a' or 'r+', got '%s'" % mode)
_ensure_nc_success(ierr, err_cls=IOError, filename=path)
@@ -2399,9 +2445,9 @@ version 4.1.2 or higher of the netcdf C lib, and rebuild netcdf4-python."""
raise ValueError(msg)
def __repr__(self):
- return self.__unicode__()
+ return self.__str__()
- def __unicode__(self):
+ def __str__(self):
ncdump = [repr(type(self))]
dimnames = tuple(_tostr(dimname)+'(%s)'%len(self.dimensions[dimname])\
for dimname in self.dimensions.keys())
@@ -2610,14 +2656,19 @@ datatype."""
enum_dict)
return self.enumtypes[datatype_name]
- def createVariable(self, varname, datatype, dimensions=(), zlib=False,
- complevel=4, shuffle=True, fletcher32=False, contiguous=False,
+ def createVariable(self, varname, datatype, dimensions=(),
+ compression=None, zlib=False,
+ complevel=4, shuffle=True,
+ szip_coding='nn',szip_pixels_per_block=8,
+ blosc_shuffle=1,fletcher32=False, contiguous=False,
chunksizes=None, endian='native', least_significant_digit=None,
- fill_value=None, chunk_cache=None):
+ significant_digits=None,quantize_mode='BitGroom',fill_value=None, chunk_cache=None):
"""
-**`createVariable(self, varname, datatype, dimensions=(), zlib=False,
+**`createVariable(self, varname, datatype, dimensions=(), compression=None, zlib=False,
complevel=4, shuffle=True, fletcher32=False, contiguous=False, chunksizes=None,
-endian='native', least_significant_digit=None, fill_value=None, chunk_cache=None)`**
+szip_coding='nn', szip_pixels_per_block=8, blosc_shuffle=1,
+endian='native', least_significant_digit=None, significant_digits=None, quantize_mode='BitGroom',
+fill_value=None, chunk_cache=None)`**
Creates a new variable with the given `varname`, `datatype`, and
`dimensions`. If dimensions are not given, the variable is assumed to be
@@ -2644,35 +2695,58 @@ length greater than one are aliases for `str`.
Data from netCDF variables is presented to python as numpy arrays with
the corresponding data type.
-`dimensions` must be a tuple containing dimension names (strings) that
-have been defined previously using `Dataset.createDimension`. The default value
+`dimensions` must be a tuple containing `Dimension` instances and/or
+dimension names (strings) that have been defined
+previously using `Dataset.createDimension`. The default value
is an empty tuple, which means the variable is a scalar.
+If the optional keyword argument `compression` is set, the data will be
+compressed in the netCDF file using the specified compression algorithm.
+Currently `zlib`,`szip`,`zstd`,`bzip2`,`blosc_lz`,`blosc_lz4`,`blosc_lz4hc`,
+`blosc_zlib` and `blosc_zstd` are supported.
+Default is `None` (no compression). All of the compressors except
+`zlib` and `szip` use the HDF5 plugin architecture.
+
If the optional keyword `zlib` is `True`, the data will be compressed in
-the netCDF file using gzip compression (default `False`).
+the netCDF file using zlib compression (default `False`). The use of this option is
+deprecated in favor of `compression='zlib'`.
-The optional keyword `complevel` is an integer between 1 and 9 describing
-the level of compression desired (default 4). Ignored if `zlib=False`.
+The optional keyword `complevel` is an integer between 0 and 9 describing
+the level of compression desired (default 4). Ignored if `compression=None`.
+A value of zero disables compression.
If the optional keyword `shuffle` is `True`, the HDF5 shuffle filter
-will be applied before compressing the data (default `True`). This
+will be applied before compressing the data with zlib (default `True`). This
significantly improves compression. Default is `True`. Ignored if
`zlib=False`.
+The optional kwarg `blosc_shuffle`is ignored
+unless the blosc compressor is used. `blosc_shuffle` can be 0 (no shuffle),
+1 (byte-wise shuffle) or 2 (bit-wise shuffle). Default is 1.
+
+The optional kwargs `szip_coding` and `szip_pixels_per_block` are ignored
+unless the szip compressor is used. `szip_coding` can be `ec` (entropy coding)
+or `nn` (nearest neighbor coding). Default is `nn`. `szip_pixels_per_block`
+can be 4, 8, 16 or 32 (default 8).
+
If the optional keyword `fletcher32` is `True`, the Fletcher32 HDF5
checksum algorithm is activated to detect errors. Default `False`.
If the optional keyword `contiguous` is `True`, the variable data is
stored contiguously on disk. Default `False`. Setting to `True` for
a variable with an unlimited dimension will trigger an error.
+Fixed size variables (with no unlimited dimension) with no compression filters
+are contiguous by default.
The optional keyword `chunksizes` can be used to manually specify the
-HDF5 chunksizes for each dimension of the variable. A detailed
-discussion of HDF chunking and I/O performance is available
-[here](http://www.hdfgroup.org/HDF5/doc/H5.user/Chunking.html).
+HDF5 chunksizes for each dimension of the variable.
+A detailed discussion of HDF chunking and I/O performance is available
+[here](https://support.hdfgroup.org/HDF5/doc/Advanced/Chunking).
+The default chunking scheme in the netcdf-c library is discussed
+[here](https://www.unidata.ucar.edu/software/netcdf/documentation/NUG/netcdf_perf_chunking.html).
Basically, you want the chunk size for each dimension to match as
closely as possible the size of the data block that users will read
-from the file. `chunksizes` cannot be set if `contiguous=True`.
+from the file. `chunksizes` cannot be set if `contiguous=True`.
The optional keyword `endian` can be used to control whether the
data is stored in little or big endian format on disk. Possible
@@ -2682,17 +2756,14 @@ but if the data is always going to be read on a computer with the
opposite format as the one used to create the file, there may be
some performance advantage to be gained by setting the endian-ness.
-The `zlib, complevel, shuffle, fletcher32, contiguous, chunksizes` and `endian`
-keywords are silently ignored for netCDF 3 files that do not use HDF5.
-
The optional keyword `fill_value` can be used to override the default
netCDF `_FillValue` (the value that the variable gets filled with before
any data is written to it, defaults given in the dict `netCDF4.default_fillvals`).
If fill_value is set to `False`, then the variable is not pre-filled.
-If the optional keyword parameter `least_significant_digit` is
+If the optional keyword parameters `least_significant_digit` or `significant_digits` are
specified, variable data will be truncated (quantized). In conjunction
-with `zlib=True` this produces 'lossy', but significantly more
+with `compression='zlib'` this produces 'lossy', but significantly more
efficient compression. For example, if `least_significant_digit=1`,
data will be quantized using `numpy.around(scale*data)/scale`, where
scale = 2**bits, and bits is determined so that a precision of 0.1 is
@@ -2700,7 +2771,14 @@ retained (in this case bits=4). From the
[PSL metadata conventions](http://www.esrl.noaa.gov/psl/data/gridded/conventions/cdc_netcdf_standard.shtml):
"least_significant_digit -- power of ten of the smallest decimal place
in unpacked data that is a reliable value." Default is `None`, or no
-quantization, or 'lossless' compression.
+quantization, or 'lossless' compression. If `significant_digits=3`
+then the data will be quantized so that three significant digits are retained, independent
+of the floating point exponent. The keyword argument `quantize_mode` controls
+the quantization algorithm (default 'BitGroom', 'BitRound' and
+'GranularBitRound' also available). The 'GranularBitRound'
+algorithm may result in better compression for typical geophysical datasets.
+This `significant_digits` kwarg is only available with netcdf-c >= 4.9.0, and
+only works with `NETCDF4` or `NETCDF4_CLASSIC` formatted files.
When creating variables in a `NETCDF4` or `NETCDF4_CLASSIC` formatted file,
HDF5 creates something called a 'chunk cache' for each variable. The
@@ -2734,7 +2812,7 @@ string containing the name of the Variable instance.
The `least_significant_digit`
attributes describes the power of ten of the smallest decimal place in
the data the contains a reliable value. assigned to the `Variable`
-instance. If `None`, the data is not truncated. The `ndim` attribute
+instance. The `ndim` attribute
is the number of variable dimensions."""
# if varname specified as a path, split out group names.
varname = posixpath.normpath(varname)
@@ -2744,12 +2822,26 @@ is the number of variable dimensions."""
group = self
else:
group = self.createGroup(dirname)
+ # if dimensions is a single string or Dimension instance,
+ # convert to a tuple.
+ # This prevents a common error that occurs when
+ # dimensions = 'lat' instead of ('lat',)
+ if isinstance(dimensions, (str, bytes, Dimension)):
+ dimensions = dimensions,
+ # convert elements of dimensions tuple to Dimension
+ # instances if they are strings.
+ # _find_dim looks for dimension in this group, and if not
+ # found there, looks in parent (and it's parent, etc, back to root).
+ dimensions =\
+ tuple(_find_dim(group,d) if isinstance(d,(str,bytes)) else d for d in dimensions)
# create variable.
group.variables[varname] = Variable(group, varname, datatype,
- dimensions=dimensions, zlib=zlib, complevel=complevel, shuffle=shuffle,
+ dimensions=dimensions, compression=compression, zlib=zlib, complevel=complevel, shuffle=shuffle,
+ szip_coding=szip_coding, szip_pixels_per_block=szip_pixels_per_block,
+ blosc_shuffle=blosc_shuffle,
fletcher32=fletcher32, contiguous=contiguous, chunksizes=chunksizes,
endian=endian, least_significant_digit=least_significant_digit,
- fill_value=fill_value, chunk_cache=chunk_cache)
+ significant_digits=significant_digits,quantize_mode=quantize_mode,fill_value=fill_value, chunk_cache=chunk_cache)
return group.variables[varname]
def renameVariable(self, oldname, newname):
@@ -3441,9 +3533,9 @@ Read-only class variables:
raise AttributeError("size cannot be altered")
def __repr__(self):
- return self.__unicode__()
+ return self.__str__()
- def __unicode__(self):
+ def __str__(self):
if not dir(self._grp):
return 'Dimension object no longer valid'
if self.isunlimited():
@@ -3551,6 +3643,23 @@ smallest decimal place in the data the contains a reliable value. Data is
truncated to this decimal place when it is assigned to the `Variable`
instance. If `None`, the data is not truncated.
+**`significant_digits`**: New in version 1.6.0. Describes the number of significant
+digits in the data the contains a reliable value. Data is
+truncated to retain this number of significant digits when it is assigned to the
+`Variable` instance. If `None`, the data is not truncated.
+Only available with netcdf-c >= 4.9.0,
+and only works with `NETCDF4` or `NETCDF4_CLASSIC` formatted files.
+The number of significant digits used in the quantization of variable data can be
+obtained using the `Variable.significant_digits` method. Default `None` -
+no quantization done.
+
+**`quantize_mode`**: New in version 1.6.0. Controls
+the quantization algorithm (default 'BitGroom', 'BitRound' and
+'GranularBitRound' also available). The 'GranularBitRound'
+algorithm may result in better compression for typical geophysical datasets.
+Ignored if `significant_digits` not specified. If 'BitRound' is used, then
+`significant_digits` is interpreted as binary (not decimal) digits.
+
**`__orthogonal_indexing__`**: Always `True`. Indicates to client code
that the object supports 'orthogonal indexing', which means that slices
that are 1d arrays or lists slice along each dimension independently. This
@@ -3568,13 +3677,17 @@ behavior is similar to Fortran or Matlab, but different than numpy.
_iscompound, _isvlen, _isenum, _grp, _cmptype, _vltype, _enumtype,\
__orthogonal_indexing__, _has_lsd, _use_get_vars, _ncstring_attrs__
- def __init__(self, grp, name, datatype, dimensions=(), zlib=False,
- complevel=4, shuffle=True, fletcher32=False, contiguous=False,
+ def __init__(self, grp, name, datatype, dimensions=(),
+ compression=None, zlib=False,
+ complevel=4, shuffle=True, szip_coding='nn', szip_pixels_per_block=8,
+ blosc_shuffle=1,
+ fletcher32=False, contiguous=False,
chunksizes=None, endian='native', least_significant_digit=None,
- fill_value=None, chunk_cache=None, **kwargs):
+ significant_digits=None,quantize_mode='BitGroom',fill_value=None, chunk_cache=None, **kwargs):
"""
- **`__init__(self, group, name, datatype, dimensions=(), zlib=False,
- complevel=4, shuffle=True, fletcher32=False, contiguous=False,
+ **`__init__(self, group, name, datatype, dimensions=(), compression=None, zlib=False,
+ complevel=4, shuffle=True, szip_coding='nn', szip_pixels_per_block=8,
+ blosc_shuffle=1, fletcher32=False, contiguous=False,
chunksizes=None, endian='native',
least_significant_digit=None,fill_value=None,chunk_cache=None)`**
@@ -3603,31 +3716,54 @@ behavior is similar to Fortran or Matlab, but different than numpy.
(for a variable-length string array). Numpy string and unicode datatypes with
length greater than one are aliases for `str`.
- **`dimensions`**: a tuple containing the variable's dimension names
+ **`dimensions`**: a tuple containing the variable's Dimension instances
(defined previously with `createDimension`). Default is an empty tuple
which means the variable is a scalar (and therefore has no dimensions).
+ **`compression`**: compression algorithm to use.
+ Currently `zlib`,`szip`,`zstd`,`bzip2`,`blosc_lz`,`blosc_lz4`,`blosc_lz4hc`,
+ `blosc_zlib` and `blosc_zstd` are supported.
+ Default is `None` (no compression). All of the compressors except
+ `zlib` and `szip` use the HDF5 plugin architecture.
+
**`zlib`**: if `True`, data assigned to the `Variable`
- instance is compressed on disk. Default `False`.
+ instance is compressed on disk. Default `False`. Deprecated - use
+ `compression='zlib'` instead.
- **`complevel`**: the level of zlib compression to use (1 is the fastest,
+ **`complevel`**: the level of compression to use (1 is the fastest,
but poorest compression, 9 is the slowest but best compression). Default 4.
- Ignored if `zlib=False`.
+ Ignored if `compression=None` or `szip`. A value of 0 disables compression.
**`shuffle`**: if `True`, the HDF5 shuffle filter is applied
- to improve compression. Default `True`. Ignored if `zlib=False`.
+ to improve zlib compression. Default `True`. Ignored unless `compression = 'zlib'`.
+
+ **`blosc_shuffle`**: shuffle filter inside blosc compressor (only
+ relevant if compression kwarg set to one of the blosc compressors).
+ Can be 0 (no blosc shuffle), 1 (bytewise shuffle) or 2 (bitwise
+ shuffle)). Default is 1. Ignored if blosc compressor not used.
+
+ **`szip_coding`**: szip coding method. Can be `ec` (entropy coding)
+ or `nn` (nearest neighbor coding). Default is `nn`.
+ Ignored if szip compressor not used.
+
+ **`szip_pixels_per_block`**: Can be 4,8,16 or 32 (Default 8).
+ Ignored if szip compressor not used.
**`fletcher32`**: if `True` (default `False`), the Fletcher32 checksum
algorithm is used for error detection.
**`contiguous`**: if `True` (default `False`), the variable data is
stored contiguously on disk. Default `False`. Setting to `True` for
- a variable with an unlimited dimension will trigger an error.
+ a variable with an unlimited dimension will trigger an error. Fixed
+ size variables (with no unlimited dimension) with no compression
+ filters are contiguous by default.
**`chunksizes`**: Can be used to specify the HDF5 chunksizes for each
dimension of the variable. A detailed discussion of HDF chunking and I/O
performance is available
- [here](http://www.hdfgroup.org/HDF5/doc/H5.user/Chunking.html).
+ [here](https://support.hdfgroup.org/HDF5/doc/Advanced/Chunking).
+ The default chunking scheme in the netcdf-c library is discussed
+ [here](https://www.unidata.ucar.edu/software/netcdf/documentation/NUG/netcdf_perf_chunking.html).
Basically, you want the chunk size for each dimension to match as
closely as possible the size of the data block that users will read
from the file. `chunksizes` cannot be set if `contiguous=True`.
@@ -3641,32 +3777,47 @@ behavior is similar to Fortran or Matlab, but different than numpy.
some performance advantage to be gained by setting the endian-ness.
For netCDF 3 files (that don't use HDF5), only `endian='native'` is allowed.
- The `zlib, complevel, shuffle, fletcher32, contiguous` and `chunksizes`
+ The `compression, zlib, complevel, shuffle, fletcher32, contiguous` and `chunksizes`
keywords are silently ignored for netCDF 3 files that do not use HDF5.
- **`least_significant_digit`**: If specified, variable data will be
- truncated (quantized). In conjunction with `zlib=True` this produces
+ **`least_significant_digit`**: If this or `significant_digits` are specified,
+ variable data will be truncated (quantized).
+ In conjunction with `compression='zlib'` this produces
'lossy', but significantly more efficient compression. For example, if
`least_significant_digit=1`, data will be quantized using
around(scale*data)/scale, where scale = 2**bits, and bits is determined
so that a precision of 0.1 is retained (in this case bits=4). Default is
`None`, or no quantization.
+ **`significant_digits`**: New in version 1.6.0.
+ As described for `least_significant_digit`
+ except the number of significant digits retained is prescribed independent
+ of the floating point exponent. Default `None` - no quantization done.
+
+ **`quantize_mode`**: New in version 1.6.0. Controls
+ the quantization algorithm (default 'BitGroom', 'BitRound' and
+ 'GranularBitRound' also available). The 'GranularBitRound'
+ algorithm may result in better compression for typical geophysical datasets.
+ Ignored if `significant_digts` not specified. If 'BitRound' is used, then
+ `significant_digits` is interpreted as binary (not decimal) digits.
+
**`fill_value`**: If specified, the default netCDF `_FillValue` (the
value that the variable gets filled with before any data is written to it)
is replaced with this value. If fill_value is set to `False`, then
the variable is not pre-filled. The default netCDF fill values can be found
in the dictionary `netCDF4.default_fillvals`.
-
+
**`chunk_cache`**: If specified, sets the chunk cache size for this variable.
- Persists as long as Dataset is open. Use `set_var_chunk_cache` to
- change it when Dataset is re-opened.
+ Persists as long as Dataset is open. Use `set_var_chunk_cache` to
+ change it when Dataset is re-opened.
***Note***: `Variable` instances should be created using the
`Dataset.createVariable` method of a `Dataset` or
`Group` instance, not using this class directly.
"""
- cdef int ierr, ndims, icontiguous, ideflate_level, numdims, _grpid
+ cdef int ierr, ndims, icontiguous, icomplevel, numdims, _grpid, nsd,
+ cdef unsigned int iblosc_complevel,iblosc_blocksize,iblosc_compressor,iblosc_shuffle
+ cdef int iszip_coding, iszip_pixels_per_block
cdef char namstring[NC_MAX_NAME+1]
cdef char *varname
cdef nc_type xtype
@@ -3676,14 +3827,48 @@ behavior is similar to Fortran or Matlab, but different than numpy.
cdef float preemptionp
# flag to indicate that orthogonal indexing is supported
self.__orthogonal_indexing__ = True
- # if complevel is set to zero, set zlib to False.
+ # For backwards compatibility, deprecated zlib kwarg takes
+ # precedence if compression kwarg not set.
+ if zlib and not compression:
+ compression = 'zlib'
+ # if complevel is set to zero, turn off compression
if not complevel:
- zlib = False
- # if dimensions is a string, convert to a tuple
- # this prevents a common error that occurs when
- # dimensions = 'lat' instead of ('lat',)
- if type(dimensions) == str or type(dimensions) == bytes or type(dimensions) == unicode:
- dimensions = dimensions,
+ compression = None
+ zlib = False
+ szip = False
+ zstd = False
+ bzip2 = False
+ blosc_lz = False
+ blosc_lz4 = False
+ blosc_lz4hc = False
+ #blosc_snappy = False
+ blosc_zlib = False
+ blosc_zstd = False
+ if compression == 'zlib':
+ zlib = True
+ elif compression == 'szip':
+ szip = True
+ elif compression == 'zstd':
+ zstd = True
+ elif compression == 'bzip2':
+ bzip2 = True
+ elif compression == 'blosc_lz':
+ blosc_lz = True
+ elif compression == 'blosc_lz4':
+ blosc_lz4 = True
+ elif compression == 'blosc_lz4hc':
+ blosc_lz4hc = True
+ #elif compression == 'blosc_snappy':
+ # blosc_snappy = True
+ elif compression == 'blosc_zlib':
+ blosc_zlib = True
+ elif compression == 'blosc_zstd':
+ blosc_zstd = True
+ elif not compression:
+ compression = None # if compression evaluates to False, set to None.
+ pass
+ else:
+ raise ValueError("Unsupported value for compression kwarg")
self._grpid = grp._grpid
# make a weakref to group to avoid circular ref (issue 218)
# keep strong reference the default behaviour (issue 251)
@@ -3767,17 +3952,9 @@ behavior is similar to Fortran or Matlab, but different than numpy.
ndims = len(dimensions)
# find dimension ids.
if ndims:
- dims = []
dimids = <int *>malloc(sizeof(int) * ndims)
for n from 0 <= n < ndims:
- dimname = dimensions[n]
- # look for dimension in this group, and if not
- # found there, look in parent (and it's parent, etc, back to root).
- dim = _find_dim(grp, dimname)
- if dim is None:
- raise KeyError("dimension %s not defined in group %s or any group in it's family tree" % (dimname, grp.path))
- dimids[n] = dim._dimid
- dims.append(dim)
+ dimids[n] = dimensions[n]._dimid
# go into define mode if it's a netCDF 3 compatible
# file format. Be careful to exit define mode before
# any exceptions are raised.
@@ -3806,23 +3983,82 @@ behavior is similar to Fortran or Matlab, but different than numpy.
if ierr != NC_NOERR:
if grp.data_model != 'NETCDF4': grp._enddef()
_ensure_nc_success(ierr)
- # set zlib, shuffle, chunking, fletcher32 and endian
+ # set compression, shuffle, chunking, fletcher32 and endian
# variable settings.
# don't bother for NETCDF3* formats.
- # for NETCDF3* formats, the zlib,shuffle,chunking,
- # and fletcher32 are silently ignored. Only
+ # for NETCDF3* formats, the comopression,zlib,shuffle,chunking,
+ # and fletcher32 flags are silently ignored. Only
# endian='native' allowed for NETCDF3.
if grp.data_model in ['NETCDF4','NETCDF4_CLASSIC']:
- # set zlib and shuffle parameters.
- if zlib and ndims: # don't bother for scalar variable
- ideflate_level = complevel
- if shuffle:
- ierr = nc_def_var_deflate(self._grpid, self._varid, 1, 1, ideflate_level)
- else:
- ierr = nc_def_var_deflate(self._grpid, self._varid, 0, 1, ideflate_level)
- if ierr != NC_NOERR:
- if grp.data_model != 'NETCDF4': grp._enddef()
- _ensure_nc_success(ierr)
+ # set compression and shuffle parameters.
+ if compression is not None and ndims: # don't bother for scalar variable
+ if zlib:
+ icomplevel = complevel
+ if shuffle:
+ ierr = nc_def_var_deflate(self._grpid, self._varid, 1, 1, icomplevel)
+ else:
+ ierr = nc_def_var_deflate(self._grpid, self._varid, 0, 1, icomplevel)
+ if ierr != NC_NOERR:
+ if grp.data_model != 'NETCDF4': grp._enddef()
+ _ensure_nc_success(ierr)
+ if szip:
+ IF HAS_SZIP_SUPPORT:
+ try:
+ iszip_coding = _szip_dict[szip_coding]
+ except KeyError:
+ msg="unknown szip coding ('ec' or 'nn' supported)"
+ raise ValueError(msg)
+ iszip_pixels_per_block = szip_pixels_per_block
+ ierr = nc_def_var_szip(self._grpid, self._varid, iszip_coding, iszip_pixels_per_block)
+ if ierr != NC_NOERR:
+ if grp.data_model != 'NETCDF4': grp._enddef()
+ _ensure_nc_success(ierr)
+ ELSE:
+ msg = """
+compression='szip' only works if linked version of hdf5 has szip functionality enabled"""
+ raise ValueError(msg)
+ if zstd:
+ IF HAS_ZSTANDARD_SUPPORT:
+ icomplevel = complevel
+ ierr = nc_def_var_zstandard(self._grpid, self._varid, icomplevel)
+ if ierr != NC_NOERR:
+ if grp.data_model != 'NETCDF4': grp._enddef()
+ _ensure_nc_success(ierr)
+ ELSE:
+ msg = """
+compression='zstd' only works with netcdf-c >= 4.9.0. To enable, install Cython, make sure you have
+version 4.9.0 or higher netcdf-c with zstandard support, and rebuild netcdf4-python."""
+ raise ValueError(msg)
+ if bzip2:
+ IF HAS_BZIP2_SUPPORT:
+ icomplevel = complevel
+ ierr = nc_def_var_bzip2(self._grpid, self._varid, icomplevel)
+ if ierr != NC_NOERR:
+ if grp.data_model != 'NETCDF4': grp._enddef()
+ _ensure_nc_success(ierr)
+ ELSE:
+ msg = """
+compression='bzip2' only works with netcdf-c >= 4.9.0. To enable, install Cython, make sure you have
+version 4.9.0 or higher netcdf-c with bzip2 support, and rebuild netcdf4-python."""
+ raise ValueError(msg)
+ if blosc_zstd or blosc_lz or blosc_lz4 or blosc_lz4hc or blosc_zlib:
+ IF HAS_BLOSC_SUPPORT:
+ iblosc_compressor = _blosc_dict[compression]
+ iblosc_shuffle = blosc_shuffle
+ iblosc_blocksize = 0 # not currently used by c lib
+ iblosc_complevel = complevel
+ ierr = nc_def_var_blosc(self._grpid, self._varid,\
+ iblosc_compressor,\
+ iblosc_complevel,iblosc_blocksize,\
+ iblosc_shuffle)
+ if ierr != NC_NOERR:
+ if grp.data_model != 'NETCDF4': grp._enddef()
+ _ensure_nc_success(ierr)
+ ELSE:
+ msg = """
+compression='blosc_*' only works with netcdf-c >= 4.9.0. To enable, install Cython, make sure you have
+version 4.9.0 or higher netcdf-c with blosc support, and rebuild netcdf4-python."""
+ raise ValueError(msg)
# set checksum.
if fletcher32 and ndims: # don't bother for scalar variable
ierr = nc_def_var_fletcher32(self._grpid, self._varid, 1)
@@ -3845,8 +4081,8 @@ behavior is similar to Fortran or Matlab, but different than numpy.
raise ValueError('chunksizes must be a sequence with the same length as dimensions')
chunksizesp = <size_t *>malloc(sizeof(size_t) * ndims)
for n from 0 <= n < ndims:
- if not dims[n].isunlimited() and \
- chunksizes[n] > dims[n].size:
+ if not dimensions[n].isunlimited() and \
+ chunksizes[n] > dimensions[n].size:
msg = 'chunksize cannot exceed dimension size'
raise ValueError(msg)
chunksizesp[n] = chunksizes[n]
@@ -3865,6 +4101,29 @@ behavior is similar to Fortran or Matlab, but different than numpy.
pass # this is the default format.
else:
raise ValueError("'endian' keyword argument must be 'little','big' or 'native', got '%s'" % endian)
+ # set quantization
+ IF HAS_QUANTIZATION_SUPPORT:
+ if significant_digits is not None:
+ nsd = significant_digits
+ if quantize_mode == 'BitGroom':
+ ierr = nc_def_var_quantize(self._grpid,
+ self._varid, NC_QUANTIZE_BITGROOM, nsd)
+ elif quantize_mode == 'GranularBitRound':
+ ierr = nc_def_var_quantize(self._grpid,
+ self._varid, NC_QUANTIZE_GRANULARBR, nsd)
+ elif quantize_mode == 'BitRound':
+ ierr = nc_def_var_quantize(self._grpid,
+ self._varid, NC_QUANTIZE_BITROUND, nsd)
+ else:
+ raise ValueError("'quantize_mode' keyword argument must be 'BitGroom','GranularBitRound' or 'BitRound', got '%s'" % quantize_mode)
+
+ ELSE:
+ if significant_digits is not None:
+ msg = """
+significant_digits kwarg only works with netcdf-c >= 4.9.0. To enable, install Cython, make sure you have
+version 4.9.0 or higher netcdf-c, and rebuild netcdf4-python. Otherwise, use least_significant_digit
+kwarg for quantization."""
+ raise ValueError(msg)
if ierr != NC_NOERR:
if grp.data_model != 'NETCDF4': grp._enddef()
_ensure_nc_success(ierr)
@@ -3906,9 +4165,7 @@ behavior is similar to Fortran or Matlab, but different than numpy.
if grp.data_model != 'NETCDF4': grp._enddef()
# count how many unlimited dimensions there are.
self._nunlimdim = 0
- for dimname in dimensions:
- # look in current group, and parents for dim.
- dim = _find_dim(self._grp, dimname)
+ for dim in dimensions:
if dim.isunlimited(): self._nunlimdim = self._nunlimdim + 1
# set ndim attribute (number of dimensions).
with nogil:
@@ -3947,9 +4204,9 @@ behavior is similar to Fortran or Matlab, but different than numpy.
return self[...]
def __repr__(self):
- return self.__unicode__()
+ return self.__str__()
- def __unicode__(self):
+ def __str__(self):
cdef int ierr, no_fill
if not dir(self._grp):
return 'Variable object no longer valid'
@@ -4183,24 +4440,100 @@ attributes."""
**`filters(self)`**
return dictionary containing HDF5 filter parameters."""
- cdef int ierr,ideflate,ishuffle,ideflate_level,ifletcher32
- filtdict = {'zlib':False,'shuffle':False,'complevel':0,'fletcher32':False}
+ cdef int ierr,ideflate,ishuffle,icomplevel,icomplevel_zstd,icomplevel_bzip2,ifletcher32
+ cdef int izstd=0
+ cdef int ibzip2=0
+ cdef int iblosc=0
+ cdef int iszip=0
+ cdef unsigned int iblosc_complevel,iblosc_blocksize,iblosc_compressor,iblosc_shuffle
+ cdef int iszip_coding, iszip_pixels_per_block
+ filtdict = {'zlib':False,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':False,'complevel':0,'fletcher32':False}
if self._grp.data_model not in ['NETCDF4_CLASSIC','NETCDF4']: return
with nogil:
- ierr = nc_inq_var_deflate(self._grpid, self._varid, &ishuffle, &ideflate, &ideflate_level)
+ ierr = nc_inq_var_deflate(self._grpid, self._varid, &ishuffle, &ideflate, &icomplevel)
_ensure_nc_success(ierr)
with nogil:
ierr = nc_inq_var_fletcher32(self._grpid, self._varid, &ifletcher32)
_ensure_nc_success(ierr)
+ IF HAS_ZSTANDARD_SUPPORT:
+ ierr = nc_inq_var_zstandard(self._grpid, self._varid, &izstd,\
+ &icomplevel_zstd)
+ if ierr != 0: izstd=0
+ # _ensure_nc_success(ierr)
+ IF HAS_BZIP2_SUPPORT:
+ ierr = nc_inq_var_bzip2(self._grpid, self._varid, &ibzip2,\
+ &icomplevel_bzip2)
+ if ierr != 0: ibzip2=0
+ #_ensure_nc_success(ierr)
+ IF HAS_BLOSC_SUPPORT:
+ ierr = nc_inq_var_blosc(self._grpid, self._varid, &iblosc,\
+ &iblosc_compressor,&iblosc_complevel,&iblosc_blocksize,&iblosc_shuffle)
+ if ierr != 0: iblosc=0
+ #_ensure_nc_success(ierr)
+ IF HAS_SZIP_SUPPORT:
+ ierr = nc_inq_var_szip(self._grpid, self._varid, &iszip_coding,\
+ &iszip_pixels_per_block)
+ if ierr != 0:
+ iszip=0
+ else:
+ if iszip_coding:
+ iszip=1
+ else:
+ iszip=0
+ #_ensure_nc_success(ierr)
if ideflate:
filtdict['zlib']=True
- filtdict['complevel']=ideflate_level
+ filtdict['complevel']=icomplevel
+ if izstd:
+ filtdict['zstd']=True
+ filtdict['complevel']=icomplevel_zstd
+ if ibzip2:
+ filtdict['bzip2']=True
+ filtdict['complevel']=icomplevel_bzip2
+ if iblosc:
+ blosc_compressor = iblosc_compressor
+ filtdict['blosc']={'compressor':_blosc_dict_inv[blosc_compressor],'shuffle':iblosc_shuffle}
+ filtdict['complevel']=iblosc_complevel
+ if iszip:
+ szip_coding = iszip_coding
+ filtdict['szip']={'coding':_szip_dict_inv[szip_coding],'pixels_per_block':iszip_pixels_per_block}
if ishuffle:
filtdict['shuffle']=True
if ifletcher32:
filtdict['fletcher32']=True
return filtdict
+ def quantization(self):
+ """
+**`quantization(self)`**
+
+return number of significant digits and the algorithm used in quantization.
+Returns None if quantization not active.
+"""
+ IF HAS_QUANTIZATION_SUPPORT:
+ cdef int ierr, nsd, quantize_mode
+ if self._grp.data_model not in ['NETCDF4_CLASSIC','NETCDF4']:
+ return None
+ else:
+ with nogil:
+ ierr = nc_inq_var_quantize(self._grpid, self._varid, &quantize_mode, &nsd)
+ _ensure_nc_success(ierr)
+ if quantize_mode == NC_NOQUANTIZE:
+ return None
+ else:
+ if quantize_mode == NC_QUANTIZE_GRANULARBR:
+ sig_digits = nsd
+ quant_mode = 'GranularBitRound'
+ elif quantize_mode == NC_QUANTIZE_BITROUND:
+ sig_digits = nsd # interpreted as bits, not decimal
+ quant_mode = 'BitRound'
+ else:
+ sig_digits = nsd
+ quant_mode = 'BitGroom'
+ return sig_digits, quant_mode
+ ELSE:
+ return None
+
def endian(self):
"""
**`endian(self)`**
@@ -4794,15 +5127,20 @@ rename a `Variable` attribute named `oldname` to `newname`."""
def _check_safecast(self, attname):
# check to see that variable attribute exists
# can can be safely cast to variable data type.
+ msg="""WARNING: %s not used since it
+cannot be safely cast to variable data type""" % attname
if hasattr(self, attname):
att = numpy.array(self.getncattr(attname))
else:
return False
- atta = numpy.array(att, self.dtype)
+ try:
+ atta = numpy.array(att, self.dtype)
+ except ValueError:
+ is_safe = False
+ warnings.warn(msg)
+ return is_safe
is_safe = _safecast(att,atta)
if not is_safe:
- msg="""WARNING: %s not used since it
-cannot be safely cast to variable data type""" % attname
warnings.warn(msg)
return is_safe
@@ -5542,9 +5880,9 @@ the user.
self.name = dtype_name
def __repr__(self):
- return self.__unicode__()
+ return self.__str__()
- def __unicode__(self):
+ def __str__(self):
return "%r: name = '%s', numpy dtype = %s" %\
(type(self), self.name, self.dtype)
@@ -5824,9 +6162,9 @@ the user.
self.name = dtype_name
def __repr__(self):
- return self.__unicode__()
+ return self.__str__()
- def __unicode__(self):
+ def __str__(self):
if self.dtype == str:
return '%r: string type' % (type(self),)
else:
@@ -5934,9 +6272,9 @@ the user.
self.enum_dict = enum_dict
def __repr__(self):
- return self.__unicode__()
+ return self.__str__()
- def __unicode__(self):
+ def __str__(self):
return "%r: name = '%s', numpy dtype = %s, fields/values =%s" %\
(type(self), self.name, self.dtype, self.enum_dict)
@@ -6424,6 +6762,7 @@ class _Variable:
def __getattr__(self,name):
if name == 'shape': return self._shape()
if name == 'ndim': return len(self._shape())
+ if name == 'name': return self._name
try:
return self.__dict__[name]
except:
=====================================
src/netCDF4/plugins/empty.txt
=====================================
=====================================
src/netCDF4/utils.py
=====================================
@@ -45,7 +45,10 @@ def _find_dim(grp, dimname):
group = group.parent
except:
raise ValueError("cannot find dimension %s in this group or parent groups" % dimname)
- return dim
+ if dim is None:
+ raise KeyError("dimension %s not defined in group %s or any group in it's family tree" % (dimname, grp.path))
+ else:
+ return dim
def _walk_grps(topgrp):
"""Iterate through all (sub-) groups of topgrp, similar to os.walktree.
=====================================
test/issue1152.nc
=====================================
Binary files /dev/null and b/test/issue1152.nc differ
=====================================
test/run_all.py
=====================================
@@ -1,7 +1,10 @@
import glob, os, sys, unittest, struct
from netCDF4 import getlibversion,__hdf5libversion__,__netcdf4libversion__,__version__
from netCDF4 import __has_cdf5_format__, __has_nc_inq_path__, __has_nc_create_mem__, \
- __has_parallel4_support__, __has_pnetcdf_support__
+ __has_parallel4_support__, __has_pnetcdf_support__, \
+ __has_zstandard_support__, __has_bzip2_support__, \
+ __has_blosc_support__,__has_quantization_support__,\
+ __has_szip_support__
# can also just run
# python -m unittest discover . 'tst*py'
@@ -20,6 +23,21 @@ if not __has_nc_create_mem__:
if not __has_cdf5_format__ or struct.calcsize("P") < 8:
test_files.remove('tst_cdf5.py')
sys.stdout.write('not running tst_cdf5.py ...\n')
+if not __has_quantization_support__:
+ test_files.remove('tst_compression_quant.py')
+ sys.stdout.write('not running tst_compression_quant.py ...\n')
+if not __has_zstandard_support__ or os.getenv('NO_PLUGINS'):
+ test_files.remove('tst_compression_zstd.py')
+ sys.stdout.write('not running tst_compression_zstd.py ...\n')
+if not __has_bzip2_support__ or os.getenv('NO_PLUGINS'):
+ test_files.remove('tst_compression_bzip2.py')
+ sys.stdout.write('not running tst_compression_bzip2.py ...\n')
+if not __has_blosc_support__ or os.getenv('NO_PLUGINS'):
+ test_files.remove('tst_compression_blosc.py')
+ sys.stdout.write('not running tst_compression_blosc.py ...\n')
+if not __has_szip_support__:
+ test_files.remove('tst_compression_szip.py')
+ sys.stdout.write('not running tst_compression_szip.py ...\n')
# Don't run tests that require network connectivity
if os.getenv('NO_NET'):
=====================================
test/tst_compression.py
=====================================
@@ -20,10 +20,25 @@ def write_netcdf(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=F
foo = file.createVariable('data',\
dtype,('n'),zlib=zlib,least_significant_digit=least_significant_digit,\
shuffle=shuffle,contiguous=contiguous,complevel=complevel,fletcher32=fletcher32,chunksizes=chunksizes)
+ # use compression kwarg instead of deprecated zlib
+ if zlib:
+ compression='zlib'
+ else:
+ compression=None
+ # anything that evaluates to False is same as None
+ #compression=False
+ #compression=''
+ #compression=0
+ #compression='gzip' # should fail
+ foo2 = file.createVariable('data2',\
+ dtype,('n'),compression=compression,least_significant_digit=least_significant_digit,\
+ shuffle=shuffle,contiguous=contiguous,complevel=complevel,fletcher32=fletcher32,chunksizes=chunksizes)
foo[:] = data
+ foo2[:] = data
file.close()
file = Dataset(filename)
data = file.variables['data'][:]
+ data2 = file.variables['data2'][:]
file.close()
def write_netcdf2(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=False,contiguous=False,\
@@ -68,18 +83,37 @@ class CompressionTestCase(unittest.TestCase):
def runTest(self):
"""testing zlib and shuffle compression filters"""
uncompressed_size = os.stat(self.files[0]).st_size
+ # check uncompressed data
+ f = Dataset(self.files[0])
+ size = os.stat(self.files[0]).st_size
+ assert_almost_equal(array,f.variables['data'][:])
+ assert_almost_equal(array,f.variables['data2'][:])
+ assert f.variables['data'].filters() ==\
+ {'zlib':False,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':False,'complevel':0,'fletcher32':False}
+ assert f.variables['data2'].filters() ==\
+ {'zlib':False,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':False,'complevel':0,'fletcher32':False}
+ assert_almost_equal(size,uncompressed_size)
+ f.close()
# check compressed data.
f = Dataset(self.files[1])
size = os.stat(self.files[1]).st_size
assert_almost_equal(array,f.variables['data'][:])
- assert f.variables['data'].filters() == {'zlib':True,'shuffle':False,'complevel':6,'fletcher32':False}
+ assert_almost_equal(array,f.variables['data2'][:])
+ assert f.variables['data'].filters() ==\
+ {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':False,'complevel':6,'fletcher32':False}
+ assert f.variables['data2'].filters() ==\
+ {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':False,'complevel':6,'fletcher32':False}
assert(size < 0.95*uncompressed_size)
f.close()
# check compression with shuffle
f = Dataset(self.files[2])
size = os.stat(self.files[2]).st_size
assert_almost_equal(array,f.variables['data'][:])
- assert f.variables['data'].filters() == {'zlib':True,'shuffle':True,'complevel':6,'fletcher32':False}
+ assert_almost_equal(array,f.variables['data2'][:])
+ assert f.variables['data'].filters() ==\
+ {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':True,'complevel':6,'fletcher32':False}
+ assert f.variables['data2'].filters() ==\
+ {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':True,'complevel':6,'fletcher32':False}
assert(size < 0.85*uncompressed_size)
f.close()
# check lossy compression without shuffle
@@ -87,12 +121,14 @@ class CompressionTestCase(unittest.TestCase):
size = os.stat(self.files[3]).st_size
checkarray = _quantize(array,lsd)
assert_almost_equal(checkarray,f.variables['data'][:])
+ assert_almost_equal(checkarray,f.variables['data2'][:])
assert(size < 0.27*uncompressed_size)
f.close()
# check lossy compression with shuffle
f = Dataset(self.files[4])
size = os.stat(self.files[4]).st_size
assert_almost_equal(checkarray,f.variables['data'][:])
+ assert_almost_equal(checkarray,f.variables['data2'][:])
assert(size < 0.20*uncompressed_size)
size_save = size
f.close()
@@ -100,7 +136,11 @@ class CompressionTestCase(unittest.TestCase):
f = Dataset(self.files[5])
size = os.stat(self.files[5]).st_size
assert_almost_equal(checkarray,f.variables['data'][:])
- assert f.variables['data'].filters() == {'zlib':True,'shuffle':True,'complevel':6,'fletcher32':True}
+ assert_almost_equal(checkarray,f.variables['data2'][:])
+ assert f.variables['data'].filters() ==\
+ {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':True,'complevel':6,'fletcher32':True}
+ assert f.variables['data2'].filters() ==\
+ {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':True,'complevel':6,'fletcher32':True}
assert(size < 0.20*uncompressed_size)
# should be slightly larger than without fletcher32
assert(size > size_save)
@@ -109,7 +149,8 @@ class CompressionTestCase(unittest.TestCase):
f = Dataset(self.files[6])
checkarray2 = _quantize(array2,lsd)
assert_almost_equal(checkarray2,f.variables['data2'][:])
- assert f.variables['data2'].filters() == {'zlib':True,'shuffle':True,'complevel':6,'fletcher32':True}
+ assert f.variables['data2'].filters() ==\
+ {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':True,'complevel':6,'fletcher32':True}
assert f.variables['data2'].chunking() == [chunk1,chunk2]
f.close()
=====================================
test/tst_compression_blosc.py
=====================================
@@ -0,0 +1,77 @@
+from numpy.random.mtrand import uniform
+from netCDF4 import Dataset
+from numpy.testing import assert_almost_equal
+import os, tempfile, unittest
+
+ndim = 100000
+iblosc_shuffle=2
+iblosc_complevel=4
+filename = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name
+datarr = uniform(size=(ndim,))
+
+def write_netcdf(filename,dtype='f8',blosc_shuffle=1,complevel=6):
+ nc = Dataset(filename,'w')
+ nc.createDimension('n', ndim)
+ foo = nc.createVariable('data',\
+ dtype,('n'),compression=None)
+ foo_lz = nc.createVariable('data_lz',\
+ dtype,('n'),compression='blosc_lz',blosc_shuffle=blosc_shuffle,complevel=complevel)
+ foo_lz4 = nc.createVariable('data_lz4',\
+ dtype,('n'),compression='blosc_lz4',blosc_shuffle=blosc_shuffle,complevel=complevel)
+ foo_lz4hc = nc.createVariable('data_lz4hc',\
+ dtype,('n'),compression='blosc_lz4hc',blosc_shuffle=blosc_shuffle,complevel=complevel)
+ foo_zlib = nc.createVariable('data_zlib',\
+ dtype,('n'),compression='blosc_zlib',blosc_shuffle=blosc_shuffle,complevel=complevel)
+ foo_zstd = nc.createVariable('data_zstd',\
+ dtype,('n'),compression='blosc_zstd',blosc_shuffle=blosc_shuffle,complevel=complevel)
+ foo_lz[:] = datarr
+ foo_lz4[:] = datarr
+ foo_lz4hc[:] = datarr
+ foo_zlib[:] = datarr
+ foo_zstd[:] = datarr
+ nc.close()
+
+class CompressionTestCase(unittest.TestCase):
+
+ def setUp(self):
+ self.filename = filename
+ write_netcdf(self.filename,complevel=iblosc_complevel,blosc_shuffle=iblosc_shuffle)
+
+ def tearDown(self):
+ # Remove the temporary files
+ os.remove(self.filename)
+
+ def runTest(self):
+ f = Dataset(self.filename)
+ assert_almost_equal(datarr,f.variables['data'][:])
+ assert f.variables['data'].filters() ==\
+ {'zlib':False,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':False,'complevel':0,'fletcher32':False}
+ assert_almost_equal(datarr,f.variables['data_lz'][:])
+ dtest = {'zlib': False, 'szip':False, 'zstd': False, 'bzip2': False, 'blosc':
+ {'compressor': 'blosc_lz', 'shuffle': iblosc_shuffle},
+ 'shuffle': False, 'complevel': iblosc_complevel, 'fletcher32': False}
+ assert f.variables['data_lz'].filters() == dtest
+ assert_almost_equal(datarr,f.variables['data_lz4'][:])
+ dtest = {'zlib': False, 'szip':False, 'zstd': False, 'bzip2': False, 'blosc':
+ {'compressor': 'blosc_lz4', 'shuffle': iblosc_shuffle},
+ 'shuffle': False, 'complevel': iblosc_complevel, 'fletcher32': False}
+ assert f.variables['data_lz4'].filters() == dtest
+ assert_almost_equal(datarr,f.variables['data_lz4hc'][:])
+ dtest = {'zlib': False, 'szip':False, 'zstd': False, 'bzip2': False, 'blosc':
+ {'compressor': 'blosc_lz4hc', 'shuffle': iblosc_shuffle},
+ 'shuffle': False, 'complevel': iblosc_complevel, 'fletcher32': False}
+ assert f.variables['data_lz4hc'].filters() == dtest
+ assert_almost_equal(datarr,f.variables['data_zlib'][:])
+ dtest = {'zlib': False, 'szip':False, 'zstd': False, 'bzip2': False, 'blosc':
+ {'compressor': 'blosc_zlib', 'shuffle': iblosc_shuffle},
+ 'shuffle': False, 'complevel': iblosc_complevel, 'fletcher32': False}
+ assert f.variables['data_zlib'].filters() == dtest
+ assert_almost_equal(datarr,f.variables['data_zstd'][:])
+ dtest = {'zlib': False, 'szip':False, 'zstd': False, 'bzip2': False, 'blosc':
+ {'compressor': 'blosc_zstd', 'shuffle': iblosc_shuffle},
+ 'shuffle': False, 'complevel': iblosc_complevel, 'fletcher32': False}
+ assert f.variables['data_zstd'].filters() == dtest
+ f.close()
+
+if __name__ == '__main__':
+ unittest.main()
=====================================
test/tst_compression_bzip2.py
=====================================
@@ -0,0 +1,52 @@
+from numpy.random.mtrand import uniform
+from netCDF4 import Dataset
+from numpy.testing import assert_almost_equal
+import os, tempfile, unittest
+
+ndim = 100000
+filename1 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name
+filename2 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name
+array = uniform(size=(ndim,))
+
+def write_netcdf(filename,dtype='f8',complevel=6):
+ nc = Dataset(filename,'w')
+ nc.createDimension('n', ndim)
+ foo = nc.createVariable('data',\
+ dtype,('n'),compression='bzip2',complevel=complevel)
+ foo[:] = array
+ nc.close()
+
+class CompressionTestCase(unittest.TestCase):
+
+ def setUp(self):
+ self.filename1 = filename1
+ self.filename2 = filename2
+ write_netcdf(self.filename1,complevel=0) # no compression
+ write_netcdf(self.filename2,complevel=4) # with compression
+
+ def tearDown(self):
+ # Remove the temporary files
+ os.remove(self.filename1)
+ os.remove(self.filename2)
+
+ def runTest(self):
+ uncompressed_size = os.stat(self.filename1).st_size
+ # check uncompressed data
+ f = Dataset(self.filename1)
+ size = os.stat(self.filename1).st_size
+ assert_almost_equal(array,f.variables['data'][:])
+ assert f.variables['data'].filters() ==\
+ {'zlib':False,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':False,'complevel':0,'fletcher32':False}
+ assert_almost_equal(size,uncompressed_size)
+ f.close()
+ # check compressed data.
+ f = Dataset(self.filename2)
+ size = os.stat(self.filename2).st_size
+ assert_almost_equal(array,f.variables['data'][:])
+ assert f.variables['data'].filters() ==\
+ {'zlib':False,'szip':False,'zstd':False,'bzip2':True,'blosc':False,'shuffle':False,'complevel':4,'fletcher32':False}
+ assert(size < 0.96*uncompressed_size)
+ f.close()
+
+if __name__ == '__main__':
+ unittest.main()
=====================================
test/tst_compression_quant.py
=====================================
@@ -0,0 +1,112 @@
+from numpy.random.mtrand import uniform
+from netCDF4 import Dataset
+from numpy.testing import assert_almost_equal
+import numpy as np
+import os, tempfile, unittest
+
+ndim = 100000
+nfiles = 7
+files = [tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name for nfile in range(nfiles)]
+data_array = uniform(size=(ndim,))
+nsd = 3
+nsb = 10 # for BitRound, use significant bits (~3.32 sig digits)
+complevel = 6
+
+def write_netcdf(filename,zlib,significant_digits,data,dtype='f8',shuffle=False,\
+ complevel=6,quantize_mode="BitGroom"):
+ file = Dataset(filename,'w')
+ file.createDimension('n', ndim)
+ foo = file.createVariable('data',\
+ dtype,('n'),zlib=zlib,significant_digits=significant_digits,\
+ shuffle=shuffle,complevel=complevel,quantize_mode=quantize_mode)
+ foo[:] = data
+ file.close()
+ file = Dataset(filename)
+ data = file.variables['data'][:]
+ file.close()
+
+class CompressionTestCase(unittest.TestCase):
+
+ def setUp(self):
+ self.files = files
+ # no compression
+ write_netcdf(self.files[0],False,None,data_array)
+ # compressed, lossless, no shuffle.
+ write_netcdf(self.files[1],True,None,data_array)
+ # compressed, lossless, with shuffle.
+ write_netcdf(self.files[2],True,None,data_array,shuffle=True)
+ # compressed, lossy, no shuffle.
+ write_netcdf(self.files[3],True,nsd,data_array)
+ # compressed, lossy, with shuffle.
+ write_netcdf(self.files[4],True,nsd,data_array,shuffle=True)
+ # compressed, lossy, with shuffle, and alternate quantization.
+ write_netcdf(self.files[5],True,nsd,data_array,quantize_mode='GranularBitRound',shuffle=True)
+ # compressed, lossy, with shuffle, and alternate quantization.
+ write_netcdf(self.files[6],True,nsb,data_array,quantize_mode='BitRound',shuffle=True)
+
+ def tearDown(self):
+ # Remove the temporary files
+ for file in self.files:
+ os.remove(file)
+
+ def runTest(self):
+ """testing zlib and shuffle compression filters"""
+ uncompressed_size = os.stat(self.files[0]).st_size
+ #print('uncompressed size = ',uncompressed_size)
+ # check compressed data.
+ f = Dataset(self.files[1])
+ size = os.stat(self.files[1]).st_size
+ #print('compressed lossless no shuffle = ',size)
+ assert_almost_equal(data_array,f.variables['data'][:])
+ assert f.variables['data'].filters() ==\
+ {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':False,'complevel':complevel,'fletcher32':False}
+ assert(size < 0.95*uncompressed_size)
+ f.close()
+ # check compression with shuffle
+ f = Dataset(self.files[2])
+ size = os.stat(self.files[2]).st_size
+ #print('compressed lossless with shuffle ',size)
+ assert_almost_equal(data_array,f.variables['data'][:])
+ assert f.variables['data'].filters() ==\
+ {'zlib':True,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':True,'complevel':complevel,'fletcher32':False}
+ assert(size < 0.85*uncompressed_size)
+ f.close()
+ # check lossy compression without shuffle
+ f = Dataset(self.files[3])
+ size = os.stat(self.files[3]).st_size
+ errmax = (np.abs(data_array-f.variables['data'][:])).max()
+ #print('compressed lossy no shuffle = ',size,' max err = ',errmax)
+ assert(f.variables['data'].quantization() == (nsd,'BitGroom'))
+ assert(errmax < 1.e-3)
+ assert(size < 0.35*uncompressed_size)
+ f.close()
+ # check lossy compression with shuffle
+ f = Dataset(self.files[4])
+ size = os.stat(self.files[4]).st_size
+ errmax = (np.abs(data_array-f.variables['data'][:])).max()
+ print('compressed lossy with shuffle and standard quantization = ',size,' max err = ',errmax)
+ assert(f.variables['data'].quantization() == (nsd,'BitGroom'))
+ assert(errmax < 1.e-3)
+ assert(size < 0.24*uncompressed_size)
+ f.close()
+ # check lossy compression with shuffle and alternate quantization
+ f = Dataset(self.files[5])
+ size = os.stat(self.files[5]).st_size
+ errmax = (np.abs(data_array-f.variables['data'][:])).max()
+ print('compressed lossy with shuffle and alternate quantization = ',size,' max err = ',errmax)
+ assert(f.variables['data'].quantization() == (nsd,'GranularBitRound'))
+ assert(errmax < 1.e-3)
+ assert(size < 0.24*uncompressed_size)
+ f.close()
+ # check lossy compression with shuffle and alternate quantization
+ f = Dataset(self.files[6])
+ size = os.stat(self.files[6]).st_size
+ errmax = (np.abs(data_array-f.variables['data'][:])).max()
+ print('compressed lossy with shuffle and alternate quantization = ',size,' max err = ',errmax)
+ assert(f.variables['data'].quantization() == (nsb,'BitRound'))
+ assert(errmax < 1.e-3)
+ assert(size < 0.24*uncompressed_size)
+ f.close()
+
+if __name__ == '__main__':
+ unittest.main()
=====================================
test/tst_compression_szip.py
=====================================
@@ -0,0 +1,42 @@
+from numpy.random.mtrand import uniform
+from netCDF4 import Dataset
+from numpy.testing import assert_almost_equal
+import os, tempfile, unittest
+
+ndim = 100000
+filename = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name
+datarr = uniform(size=(ndim,))
+
+def write_netcdf(filename,dtype='f8'):
+ nc = Dataset(filename,'w')
+ nc.createDimension('n', ndim)
+ foo = nc.createVariable('data',\
+ dtype,('n'),compression=None)
+ foo_szip = nc.createVariable('data_szip',\
+ dtype,('n'),compression='szip',szip_coding='ec',szip_pixels_per_block=32)
+ foo[:] = datarr
+ foo_szip[:] = datarr
+ nc.close()
+
+class CompressionTestCase(unittest.TestCase):
+
+ def setUp(self):
+ self.filename = filename
+ write_netcdf(self.filename)
+
+ def tearDown(self):
+ # Remove the temporary files
+ os.remove(self.filename)
+
+ def runTest(self):
+ f = Dataset(self.filename)
+ assert_almost_equal(datarr,f.variables['data'][:])
+ assert f.variables['data'].filters() ==\
+ {'zlib':False,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':False,'complevel':0,'fletcher32':False}
+ assert_almost_equal(datarr,f.variables['data_szip'][:])
+ dtest = {'zlib': False, 'szip': {'coding': 'ec', 'pixels_per_block': 32}, 'zstd': False, 'bzip2': False, 'blosc': False, 'shuffle': False, 'complevel': 0, 'fletcher32': False}
+ assert f.variables['data_szip'].filters() == dtest
+ f.close()
+
+if __name__ == '__main__':
+ unittest.main()
=====================================
test/tst_compression_zstd.py
=====================================
@@ -0,0 +1,52 @@
+from numpy.random.mtrand import uniform
+from netCDF4 import Dataset
+from numpy.testing import assert_almost_equal
+import os, tempfile, unittest
+
+ndim = 100000
+filename1 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name
+filename2 = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name
+array = uniform(size=(ndim,))
+
+def write_netcdf(filename,dtype='f8',complevel=6):
+ nc = Dataset(filename,'w')
+ nc.createDimension('n', ndim)
+ foo = nc.createVariable('data',\
+ dtype,('n'),compression='zstd',complevel=complevel)
+ foo[:] = array
+ nc.close()
+
+class CompressionTestCase(unittest.TestCase):
+
+ def setUp(self):
+ self.filename1 = filename1
+ self.filename2 = filename2
+ write_netcdf(self.filename1,complevel=0) # no compression
+ write_netcdf(self.filename2,complevel=4) # with compression
+
+ def tearDown(self):
+ # Remove the temporary files
+ os.remove(self.filename1)
+ os.remove(self.filename2)
+
+ def runTest(self):
+ uncompressed_size = os.stat(self.filename1).st_size
+ # check uncompressed data
+ f = Dataset(self.filename1)
+ size = os.stat(self.filename1).st_size
+ assert_almost_equal(array,f.variables['data'][:])
+ assert f.variables['data'].filters() ==\
+ {'zlib':False,'szip':False,'zstd':False,'bzip2':False,'blosc':False,'shuffle':False,'complevel':0,'fletcher32':False}
+ assert_almost_equal(size,uncompressed_size)
+ f.close()
+ # check compressed data.
+ f = Dataset(self.filename2)
+ size = os.stat(self.filename2).st_size
+ assert_almost_equal(array,f.variables['data'][:])
+ assert f.variables['data'].filters() ==\
+ {'zlib':False,'szip':False,'zstd':True,'bzip2':False,'blosc':False,'shuffle':False,'complevel':4,'fletcher32':False}
+ assert(size < 0.96*uncompressed_size)
+ f.close()
+
+if __name__ == '__main__':
+ unittest.main()
=====================================
test/tst_dims.py
=====================================
@@ -19,7 +19,11 @@ TIME_NAME="time"
TIME_LEN = None
TIME_LENG = None
GROUP_NAME='forecasts'
-VAR_NAME='temp'
+VAR_NAME1='temp1'
+VAR_NAME2='temp2'
+VAR_NAME3='temp3'
+VAR_NAME4='temp4'
+VAR_NAME5='temp5'
VAR_TYPE='f8'
@@ -28,11 +32,19 @@ class DimensionsTestCase(unittest.TestCase):
def setUp(self):
self.file = FILE_NAME
f = netCDF4.Dataset(self.file, 'w')
- f.createDimension(LAT_NAME,LAT_LEN)
- f.createDimension(LON_NAME,LON_LEN)
- f.createDimension(LEVEL_NAME,LEVEL_LEN)
- f.createDimension(TIME_NAME,TIME_LEN)
- f.createVariable(VAR_NAME,VAR_TYPE,(LEVEL_NAME, LAT_NAME, LON_NAME, TIME_NAME))
+ lat_dim=f.createDimension(LAT_NAME,LAT_LEN)
+ lon_dim=f.createDimension(LON_NAME,LON_LEN)
+ lev_dim=f.createDimension(LEVEL_NAME,LEVEL_LEN)
+ time_dim=f.createDimension(TIME_NAME,TIME_LEN)
+ # specify dimensions with names
+ fv1 = f.createVariable(VAR_NAME1,VAR_TYPE,(LEVEL_NAME, LAT_NAME, LON_NAME, TIME_NAME))
+ # specify dimensions with instances
+ fv2 = f.createVariable(VAR_NAME2,VAR_TYPE,(lev_dim,lat_dim,lon_dim,time_dim))
+ # specify dimensions using a mix of names and instances
+ fv3 = f.createVariable(VAR_NAME3,VAR_TYPE,(lev_dim, LAT_NAME, lon_dim, TIME_NAME))
+ # single dim instance for name (not in a tuple)
+ fv4 = f.createVariable(VAR_NAME4,VAR_TYPE,time_dim)
+ fv5 = f.createVariable(VAR_NAME5,VAR_TYPE,TIME_NAME)
g = f.createGroup(GROUP_NAME)
g.createDimension(LAT_NAME,LAT_LENG)
g.createDimension(LON_NAME,LON_LENG)
@@ -40,7 +52,7 @@ class DimensionsTestCase(unittest.TestCase):
# (did not work prior to alpha 18)
#g.createDimension(LEVEL_NAME,LEVEL_LENG)
#g.createDimension(TIME_NAME,TIME_LENG)
- g.createVariable(VAR_NAME,VAR_TYPE,(LEVEL_NAME, LAT_NAME, LON_NAME, TIME_NAME))
+ gv = g.createVariable(VAR_NAME1,VAR_TYPE,(LEVEL_NAME, LAT_NAME, LON_NAME, TIME_NAME))
f.close()
def tearDown(self):
@@ -51,7 +63,11 @@ class DimensionsTestCase(unittest.TestCase):
"""testing dimensions"""
# check dimensions in root group.
f = netCDF4.Dataset(self.file, 'r+')
- v = f.variables[VAR_NAME]
+ v1 = f.variables[VAR_NAME1]
+ v2 = f.variables[VAR_NAME2]
+ v3 = f.variables[VAR_NAME3]
+ v4 = f.variables[VAR_NAME4]
+ v5 = f.variables[VAR_NAME5]
isunlim = [dim.isunlimited() for dim in f.dimensions.values()]
dimlens = [len(dim) for dim in f.dimensions.values()]
names_check = [LAT_NAME, LON_NAME, LEVEL_NAME, TIME_NAME]
@@ -65,6 +81,15 @@ class DimensionsTestCase(unittest.TestCase):
# check that dimension names are correct.
for name in f.dimensions.keys():
self.assertTrue(name in names_check)
+ for name in v1.dimensions:
+ self.assertTrue(name in names_check)
+ for name in v2.dimensions:
+ self.assertTrue(name in names_check)
+ for name in v3.dimensions:
+ self.assertTrue(name in names_check)
+ self.assertTrue(v4.dimensions[0] == TIME_NAME)
+ self.assertTrue(v5.dimensions[0] == TIME_NAME)
+ # check that dimension lengths are correct.
# check that dimension lengths are correct.
for name,dim in f.dimensions.items():
self.assertTrue(len(dim) == lensdict[name])
@@ -75,7 +100,7 @@ class DimensionsTestCase(unittest.TestCase):
# make sure length of dimensions change correctly.
nadd1 = 2
nadd2 = 4
- v[0:nadd1,:,:,0:nadd2] = uniform(size=(nadd1,LAT_LEN,LON_LEN,nadd2))
+ v1[0:nadd1,:,:,0:nadd2] = uniform(size=(nadd1,LAT_LEN,LON_LEN,nadd2))
lensdict[LEVEL_NAME]=nadd1
lensdict[TIME_NAME]=nadd2
# check that dimension lengths are correct.
@@ -83,7 +108,7 @@ class DimensionsTestCase(unittest.TestCase):
self.assertTrue(len(dim) == lensdict[name])
# check dimensions in subgroup.
g = f.groups[GROUP_NAME]
- vg = g.variables[VAR_NAME]
+ vg = g.variables[VAR_NAME1]
isunlim = [dim.isunlimited() for dim in g.dimensions.values()]
dimlens = [len(dim) for dim in g.dimensions.values()]
names_check = [LAT_NAME, LON_NAME, LEVEL_NAME, TIME_NAME]
=====================================
test/tst_masked.py
=====================================
@@ -85,6 +85,14 @@ class PrimitiveTypesTestCase(unittest.TestCase):
var2[:] = masked_all((10,), "u1")
dataset.close()
+ # issue #1152: if missing_value is a string that can't
+ # be cast to the variable type, issue a warning instead
+ # of raising an exception when auto-converted slice to a
+ # masked array
+ dataset = netCDF4.Dataset('issue1152.nc')
+ data = dataset['v'][:]
+ dataset.close()
+
def tearDown(self):
# Remove the temporary files
os.remove(self.file)
View it on GitLab: https://salsa.debian.org/debian-gis-team/netcdf4-python/-/commit/9f65992d808eb4fb334d3df1ab88c322053e0da5
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/netcdf4-python/-/commit/9f65992d808eb4fb334d3df1ab88c322053e0da5
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20220625/2104af2a/attachment-0001.htm>
More information about the Pkg-grass-devel
mailing list