[netcdf4-python] 02/06: New upstream version 1.3.1
Bas Couwenberg
sebastic at debian.org
Tue Oct 31 07:07:18 UTC 2017
This is an automated email from the git hooks/post-receive script.
sebastic pushed a commit to branch master
in repository netcdf4-python.
commit 55c873625b71e11d93137f823078ee6c4a096ba8
Author: Bas Couwenberg <sebastic at xs4all.nl>
Date: Tue Oct 31 07:31:54 2017 +0100
New upstream version 1.3.1
---
.travis.yml | 27 +++-
Changelog | 15 ++
PKG-INFO | 2 +-
README.md | 7 +
appveyor.yml | 27 ++--
ci/travis/build-parallel-netcdf.sh | 13 ++
docs/netCDF4/index.html | 120 ++++++++++++--
examples/mpi_example.py | 35 ++++
include/mpi-compat.h | 14 ++
include/netCDF4.pxi | 16 ++
netCDF4/__init__.py | 3 +-
netCDF4/_netCDF4.pyx | 320 +++++++++++++++++++++++++++++--------
setup.cfg | 4 +-
setup.py | 57 +++++--
test/run_all.py | 12 +-
test/tst_filepath.py | 7 +
test/tst_masked5.py | 14 +-
test/tst_multifile.py | 31 +++-
18 files changed, 599 insertions(+), 125 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 05c4e5d..8c7aa5a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,8 +11,9 @@ addons:
env:
global:
- - DEPENDS="numpy cython setuptools==18.0.1"
+ - DEPENDS="numpy>=1.9.0 cython>=0.21 setuptools>=18.0"
- NO_NET=1
+ - MPI=0
python:
- "2.7"
@@ -26,7 +27,22 @@ matrix:
# Absolute minimum dependencies.
- python: 2.7
env:
- - DEPENDS="numpy==1.9.0 cython==0.19 ordereddict==1.1 setuptools==18.0"
+ - DEPENDS="numpy==1.9.0 cython==0.21 ordereddict==1.1 setuptools==18.0"
+ # test MPI
+ - python: 2.7
+ env:
+ - MPI=1
+ - CC=mpicc
+ - DEPENDS="numpy>=1.9.0 cython>=0.21 setuptools>=18.0 mpi4py>=1.3.1"
+ - NETCDF_VERSION=4.4.1.1
+ - NETCDF_DIR=$HOME
+ - PATH=${NETCDF_DIR}/bin:${PATH} # pick up nc-config here
+ addons:
+ apt:
+ packages:
+ - openmpi-bin
+ - libopenmpi-dev
+ - libhdf5-openmpi-dev
notifications:
email: false
@@ -35,9 +51,16 @@ before_install:
- pip install $DEPENDS
install:
+ - if [ $MPI -eq 1 ] ; then ci/travis/build-parallel-netcdf.sh; fi
- python setup.py build
- python setup.py install
script:
+ - |
+ if [ $MPI -eq 1 ] ; then
+ cd examples
+ mpirun -np 4 python mpi_example.py
+ cd ..
+ fi
- cd test
- python run_all.py
diff --git a/Changelog b/Changelog
index c2bf06d..22f2274 100644
--- a/Changelog
+++ b/Changelog
@@ -1,3 +1,18 @@
+ version 1.3.1 (tag v1.3.1rel)
+=============================
+ * add parallel IO capabilities. netcdf-c and hdf5 must be compiled with MPI
+ support, and mpi4py must be installed. To open a file for parallel access,
+ use `parallel=True` in `Dataset.__init__` and optionally pass the mpi4py Comm instance
+ using the `comm` kwarg and the mpi4py Info instance using the `info` kwarg.
+ IO can be toggled between collective and independent using `Variable.set_collective`.
+ See `examples/mpi_example.py`. Issue #717, pull request #716.
+ Minimum cython dependency bumped from 0.19 to 0.21.
+ * Add optional `MFTime` calendar overload to use across all files, for example,
+ `'standard'` or `'gregorian'`. If `None` (the default), check that the calendar
+ attribute is present on each variable and values are unique across files raising
+ a `ValueError` otherwise.
+ * Allow _FillValue to be set for vlen string variables (issue #730).
+
version 1.3.0 (tag v1.3.0rel)
==============================
* always search for HDF5 headers when building, even when nc-config is used
diff --git a/PKG-INFO b/PKG-INFO
index 44f8c64..d18d49a 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: netCDF4
-Version: 1.3.0
+Version: 1.3.1
Author: Jeff Whitaker
Author-email: jeffrey s whitaker at noaa gov
Home-page: https://github.com/Unidata/netcdf4-python
diff --git a/README.md b/README.md
index c0294db..38f37cd 100644
--- a/README.md
+++ b/README.md
@@ -8,6 +8,13 @@
## News
For details on the latest updates, see the [Changelog](https://github.com/Unidata/netcdf4-python/blob/master/Changelog).
+11/01/2017: Version 1.3.1 released. Parallel IO support with MPI!
+Requires that netcdf-c and hdf5 be built with MPI support, and [mpi4py](http://mpi4py.readthedocs.io/en/stable).
+To open a file for parallel access in a program running in an MPI environment
+using mpi4py, just use `parallel=True` when creating
+the `Dataset` instance. See [`examples/mpi_example.py`](https://github.com/Unidata/netcdf4-python/blob/master/examples/mpi_example.py)
+ for a demonstration. For more info, see the tutorial [section](http://unidata.github.io/netcdf4-python/#section13).
+
9/25/2017: Version [1.3.0](https://pypi.python.org/pypi/netCDF4/1.3.0) released. Bug fixes
for `netcdftime` and optimizations for reading strided slices. `encoding` kwarg added to
`Dataset.__init__` and `Dataset.filepath` to deal with oddball encodings in filename
diff --git a/appveyor.yml b/appveyor.yml
index e324dc6..9d5057f 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -34,21 +34,20 @@ install:
# Add path, activate `conda` and update conda.
- cmd: set "PATH=%CONDA_INSTALL_LOCN%\\Scripts;%CONDA_INSTALL_LOCN%\\Library\\bin;%PATH%"
- - cmd: conda update --yes --quiet conda
- - cmd: call %CONDA_INSTALL_LOCN%\Scripts\activate.bat
-
- cmd: set PYTHONUNBUFFERED=1
-
- # Ensure defaults and conda-forge channels are present.
- - cmd: conda config --set show_channel_urls true
- - cmd: conda config --remove channels defaults
- - cmd: conda config --add channels defaults
- - cmd: conda config --add channels conda-forge
-
- # Conda build tools.
- - cmd: conda install -n root --quiet --yes obvious-ci
- - cmd: obvci_install_conda_build_tools.py
- - cmd: conda info
+ - cmd: call %CONDA_INSTALL_LOCN%\Scripts\activate.bat
+ # for obvci_appveyor_python_build_env.cmd
+ - cmd: conda update --all --yes
+ - cmd: conda install anaconda-client=1.6.3 --yes
+ - cmd: conda install -c conda-forge --yes obvious-ci
+ # for msinttypes and newer stuff
+ - cmd: conda config --prepend channels conda-forge
+ - cmd: conda config --set show_channel_urls yes
+ - cmd: conda config --set always_yes true
+ # For building conda packages
+ - cmd: conda install --yes conda-build jinja2 anaconda-client
+ # this is now the downloaded conda...
+ - cmd: conda info -a
# Skip .NET project specific build phase.
build: off
diff --git a/ci/travis/build-parallel-netcdf.sh b/ci/travis/build-parallel-netcdf.sh
new file mode 100755
index 0000000..1c38eec
--- /dev/null
+++ b/ci/travis/build-parallel-netcdf.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+
+echo "Using downloaded netCDF version ${NETCDF_VERSION} with parallel capabilities enabled"
+pushd /tmp
+wget ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-${NETCDF_VERSION}.tar.gz
+tar -xzvf netcdf-${NETCDF_VERSION}.tar.gz
+pushd netcdf-${NETCDF_VERSION}
+./configure --prefix $NETCDF_DIR --enable-netcdf-4 --enable-shared --disable-dap --enable-parallel
+make -j 2
+make install
+popd
diff --git a/docs/netCDF4/index.html b/docs/netCDF4/index.html
index fe82e85..bdf78cb 100644
--- a/docs/netCDF4/index.html
+++ b/docs/netCDF4/index.html
@@ -4,7 +4,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1, minimum-scale=1" />
<title>netCDF4 API documentation</title>
- <meta name="description" content="Version 1.3.0
+ <meta name="description" content="Version 1.3.1
-------------
- - -
@@ -1246,6 +1246,7 @@ table {
<li class="mono"><a href="#netCDF4.Variable.set_auto_mask">set_auto_mask</a></li>
<li class="mono"><a href="#netCDF4.Variable.set_auto_maskandscale">set_auto_maskandscale</a></li>
<li class="mono"><a href="#netCDF4.Variable.set_auto_scale">set_auto_scale</a></li>
+ <li class="mono"><a href="#netCDF4.Variable.set_collective">set_collective</a></li>
<li class="mono"><a href="#netCDF4.Variable.set_var_chunk_cache">set_var_chunk_cache</a></li>
<li class="mono"><a href="#netCDF4.Variable.setncattr">setncattr</a></li>
<li class="mono"><a href="#netCDF4.Variable.setncattr_string">setncattr_string</a></li>
@@ -1269,7 +1270,7 @@ table {
<header id="section-intro">
<h1 class="title"><span class="name">netCDF4</span> module</h1>
- <h2>Version 1.3.0</h2>
+ <h2>Version 1.3.1</h2>
<hr />
<h1>Introduction</h1>
<p>netcdf4-python is a Python interface to the netCDF C library. </p>
@@ -1299,7 +1300,7 @@ types) are not supported.</p>
<ul>
<li>Python 2.7 or later (python 3 works too).</li>
<li><a href="http://numpy.scipy.org">numpy array module</a>, version 1.9.0 or later.</li>
-<li><a href="http://cython.org">Cython</a>, version 0.19 or later.</li>
+<li><a href="http://cython.org">Cython</a>, version 0.21 or later.</li>
<li><a href="https://pypi.python.org/pypi/setuptools">setuptools</a>, version 18.0 or
later.</li>
<li>The HDF5 C library version 1.8.4-patch1 or higher (1.8.x recommended)
@@ -1321,6 +1322,9 @@ types) are not supported.</p>
If you want <a href="http://opendap.org">OPeNDAP</a> support, add <code>--enable-dap</code>.
If you want HDF4 SD support, add <code>--enable-hdf4</code> and add
the location of the HDF4 headers and library to <code>$CPPFLAGS</code> and <code>$LDFLAGS</code>.</li>
+<li>for MPI parallel IO support, MPI-enabled versions of the HDF5 and netcdf
+ libraries are required, as is the <a href="http://mpi4py.scipy.org">mpi4py</a> python
+ module.</li>
</ul>
<h1>Install</h1>
<ul>
@@ -1337,16 +1341,16 @@ types) are not supported.</p>
<li>run <code>python setup.py build</code>, then <code>python setup.py install</code> (as root if
necessary).</li>
<li><a href="https://pip.pypa.io/en/latest/reference/pip_install.html"><code>pip install</code></a> can
- also be used, with library paths set with environment variables. To make
- this work, the <code>USE_SETUPCFG</code> environment variable must be used to tell
- setup.py not to use <code>setup.cfg</code>.
- For example, <code>USE_SETUPCFG=0 HDF5_INCDIR=/usr/include/hdf5/serial
- HDF5_LIBDIR=/usr/lib/x86_64-linux-gnu/hdf5/serial pip install</code> has been
- shown to work on an Ubuntu/Debian linux system. Similarly, environment variables
- (all capitalized) can be used to set the include and library paths for
- <code>hdf5</code>, <code>netCDF4</code>, <code>hdf4</code>, <code>szip</code>, <code>jpeg</code>, <code>curl</code> and <code>zlib</code>. If the
- libraries are installed in standard places (e.g. <code>/usr</code> or <code>/usr/local</code>),
- the environment variables do not need to be set.</li>
+ also be used, with library paths set with environment variables. To make
+ this work, the <code>USE_SETUPCFG</code> environment variable must be used to tell
+ setup.py not to use <code>setup.cfg</code>.
+ For example, <code>USE_SETUPCFG=0 HDF5_INCDIR=/usr/include/hdf5/serial
+ HDF5_LIBDIR=/usr/lib/x86_64-linux-gnu/hdf5/serial pip install</code> has been
+ shown to work on an Ubuntu/Debian linux system. Similarly, environment variables
+ (all capitalized) can be used to set the include and library paths for
+ <code>hdf5</code>, <code>netCDF4</code>, <code>hdf4</code>, <code>szip</code>, <code>jpeg</code>, <code>curl</code> and <code>zlib</code>. If the
+ libraries are installed in standard places (e.g. <code>/usr</code> or <code>/usr/local</code>),
+ the environment variables do not need to be set.</li>
<li>run the tests in the 'test' directory by running <code>python run_all.py</code>.</li>
</ul>
<h1>Tutorial</h1>
@@ -1363,6 +1367,7 @@ types) are not supported.</p>
<li><a href="#section10">Beyond homogeneous arrays of a fixed type - compound data types.</a></li>
<li><a href="#section11">Variable-length (vlen) data types.</a></li>
<li><a href="#section12">Enum data type.</a></li>
+<li><a href="#section13">Parallel IO.</a></li>
</ol>
<h2><div id='section1'>1) Creating/Opening/Closing a netCDF file.</h2>
<p>To create a netCDF file from python, you simply call the <a href="#netCDF4.Dataset"><code>Dataset</code></a>
@@ -2117,7 +2122,69 @@ specified names.</p>
</pre></div>
-<p>All of the code in this tutorial is available in <code>examples/tutorial.py</code>,
+<h2><div id='section13'>13) Parallel IO.</h2>
+<p>If MPI parallel enabled versions of netcdf and hdf5 are detected, and
+<a href="https://mpi4py.scipy.org">mpi4py</a> is installed, netcdf4-python will
+be built with parallel IO capabilities enabled. To use parallel IO,
+your program must be running in an MPI environment using
+<a href="https://mpi4py.scipy.org">mpi4py</a>.</p>
+<div class="codehilite"><pre><span></span><span class="o">>>></span> <span class="kn">from</span> <span class="nn">mpi4py</span> <span class="kn">import</span> <span class="n">MPI</span>
+<span class="o">>>></span> <span class="kn">import</span> <span class="nn">numpy</span> <span class="kn">as</span> <span class="nn">np</span>
+<span class="o">>>></span> <span class="kn">from</span> <span class="nn">netCDF4</span> <span class="kn">import</span> <span class="n">Dataset</span>
+<span class="o">>>></span> <span class="n">rank</span> <span class="o">=</span> <span class="n">MPI</span><span class="o">.</span><span class="n">COMM_WORLD</span><span class="o">.</span><span class="n">rank</span> <span class="c1"># The process ID (integer 0-3 for 4-process run)</span>
+</pre></div>
+
+
+<p>To run an MPI-based parallel program like this, you must use <code>mpiexec</code> to launch several
+parallel instances of Python (for example, using <code>mpiexec -np 4 python mpi_example.py</code>).
+The parallel features of netcdf4-python are mostly transparent -
+when a new dataset is created or an existing dataset is opened,
+use the <code>parallel</code> keyword to enable parallel access.</p>
+<div class="codehilite"><pre><span></span><span class="o">>>></span> <span class="n">nc</span> <span class="o">=</span> <span class="n">Dataset</span><span class="p">(</span><span class="s1">'parallel_tst.nc'</span><span class="p">,</span><span class="s1">'w'</span><span class="p">,</span><span class="n">parallel</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
+</pre></div>
+
+
+<p>The optional <code>comm</code> keyword may be used to specify a particular
+MPI communicator (<code>MPI_COMM_WORLD</code> is used by default). Each process (or rank)
+can now write to the file indepedently. In this example the process rank is
+written to a different variable index on each task</p>
+<div class="codehilite"><pre><span></span><span class="o">>>></span> <span class="n">d</span> <span class="o">=</span> <span class="n">nc</span><span class="o">.</span><span class="n">createDimension</span><span class="p">(</span><span class="s1">'dim'</span><span class="p">,</span><span class="mi">4</span><span class="p">)</span>
+<span class="o">>>></span> <span class="n">v</span> <span class="o">=</span> <span class="n">nc</span><span class="o">.</span><span class="n">createVariable</span><span class="p">(</span><span class="s1">'var'</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">int</span><span class="p">,</span> <span class="s1">'dim'</span><span class="p">)</span>
+<span class="o">>>></span> <span class="n">v</span><span class="p">[</span><span class="n">rank</span><span class="p">]</span> <span class="o">=</span> <span class="n">rank</span>
+<span class="o">>>></span> <span class="n">nc</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+
+<span class="o">%</span> <span class="n">ncdump</span> <span class="n">parallel_test</span><span class="o">.</span><span class="n">nc</span>
+<span class="n">netcdf</span> <span class="n">parallel_test</span> <span class="p">{</span>
+<span class="n">dimensions</span><span class="p">:</span>
+ <span class="n">dim</span> <span class="o">=</span> <span class="mi">4</span> <span class="p">;</span>
+ <span class="n">variables</span><span class="p">:</span>
+ <span class="n">int64</span> <span class="n">var</span><span class="p">(</span><span class="n">dim</span><span class="p">)</span> <span class="p">;</span>
+ <span class="n">data</span><span class="p">:</span>
+
+ <span class="n">var</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span> <span class="p">;</span>
+<span class="p">}</span>
+</pre></div>
+
+
+<p>There are two types of parallel IO, independent (the default) and collective.
+Independent IO means that each process can do IO independently. It should not
+depend on or be affected by other processes. Collective IO is a way of doing
+IO defined in the MPI-IO standard; unlike independent IO, all processes must
+participate in doing IO. To toggle back and forth between
+the two types of IO, use the <a href="#netCDF4.Variable.set_collective"><code>set_collective</code></a>
+<a href="#netCDF4.Variable"><code>Variable</code></a>method. All metadata
+operations (such as creation of groups, types, variables, dimensions, or attributes)
+are collective. There are a couple of important limitatons of parallel IO:</p>
+<ul>
+<li>If a variable has an unlimited dimension, appending data must be done in collective mode.
+ If the write is done in independent mode, the operation will fail with a
+ a generic "HDF Error".</li>
+<li>You cannot write compressed data in parallel (although
+ you can read it).</li>
+<li>You cannot use variable-length (VLEN) data types. </li>
+</ul>
+<p>All of the code in this tutorial is available in <code>examples/tutorial.py</code>, except
+the parallel IO example, which is in <code>examples/mpi_example.py</code>.
Unit tests are in the <code>test</code> directory.</p>
<p><strong>contact</strong>: Jeffrey Whitaker <a href="mailto:jeffrey.s.whitaker@noaa.gov">jeffrey.s.whitaker@noaa.gov</a></p>
<p><strong>copyright</strong>: 2008 by Jeffrey Whitaker.</p>
@@ -2700,7 +2767,14 @@ rendered unusable when the parent Dataset instance is garbage collected.</p>
<p><strong><code>memory</code></strong>: if not <code>None</code>, open file with contents taken from this block of memory.
Must be a sequence of bytes. Note this only works with "r" mode.</p>
<p><strong><code>encoding</code></strong>: encoding used to encode filename string into bytes.
-Default is None (<code>sys.getdefaultfileencoding()</code> is used).</p></div>
+Default is None (<code>sys.getdefaultfileencoding()</code> is used).</p>
+<p><strong><code>parallel</code></strong>: open for parallel access using MPI (requires mpi4py and
+parallel-enabled netcdf-c and hdf5 libraries). Default is <code>False</code>. If
+<code>True</code>, <code>comm</code> and <code>info</code> kwargs may also be specified.</p>
+<p><strong><code>comm</code></strong>: MPI_Comm object for parallel access. Default <code>None</code>, which
+means MPI_COMM_WORLD will be used. Ignored if <code>parallel=False</code>.</p>
+<p><strong><code>info</code></strong>: MPI_Info object for parallel access. Default <code>None</code>, which
+means MPI_INFO_NULL will be used. Ignored if <code>parallel=False</code>.</p></div>
<div class="source_cont">
</div>
@@ -6297,6 +6371,22 @@ data model does not have unsigned integer data types).</p>
<div class="item">
+ <div class="name def" id="netCDF4.Variable.set_collective">
+ <p>def <span class="ident">set_collective</span>(</p><p>self,True_or_False)</p>
+ </div>
+
+
+
+
+ <div class="desc"><p>turn on or off collective parallel IO access. Ignored if file is not
+open for parallel access.</p></div>
+ <div class="source_cont">
+</div>
+
+ </div>
+
+
+ <div class="item">
<div class="name def" id="netCDF4.Variable.set_var_chunk_cache">
<p>def <span class="ident">set_var_chunk_cache</span>(</p><p>self,size=None,nelems=None,preemption=None)</p>
</div>
diff --git a/examples/mpi_example.py b/examples/mpi_example.py
new file mode 100644
index 0000000..afac52d
--- /dev/null
+++ b/examples/mpi_example.py
@@ -0,0 +1,35 @@
+# to run: mpirun -np 4 python mpi_example.py
+from mpi4py import MPI
+import numpy as np
+from netCDF4 import Dataset
+rank = MPI.COMM_WORLD.rank # The process ID (integer 0-3 for 4-process run)
+nc = Dataset('parallel_test.nc', 'w', parallel=True, comm=MPI.COMM_WORLD,
+ info=MPI.Info())
+# below should work also - MPI_COMM_WORLD and MPI_INFO_NULL will be used.
+#nc = Dataset('parallel_test.nc', 'w', parallel=True)
+d = nc.createDimension('dim',4)
+v = nc.createVariable('var', np.int, 'dim')
+v[rank] = rank
+# switch to collective mode, rewrite the data.
+v.set_collective(True)
+v[rank] = rank
+nc.close()
+# reopen the file read-only, check the data
+nc = Dataset('parallel_test.nc', parallel=True, comm=MPI.COMM_WORLD,
+ info=MPI.Info())
+assert rank==nc['var'][rank]
+nc.close()
+# reopen the file in append mode, modify the data on the last rank.
+nc = Dataset('parallel_test.nc', 'a',parallel=True, comm=MPI.COMM_WORLD,
+ info=MPI.Info())
+if rank == 3: v[rank] = 2*rank
+nc.close()
+# reopen the file read-only again, check the data.
+# leave out the comm and info kwargs to check that the defaults
+# (MPI_COMM_WORLD and MPI_INFO_NULL) work.
+nc = Dataset('parallel_test.nc', parallel=True)
+if rank == 3:
+ assert 2*rank==nc['var'][rank]
+else:
+ assert rank==nc['var'][rank]
+nc.close()
diff --git a/include/mpi-compat.h b/include/mpi-compat.h
new file mode 100644
index 0000000..367c58a
--- /dev/null
+++ b/include/mpi-compat.h
@@ -0,0 +1,14 @@
+/* Author: Lisandro Dalcin */
+/* Contact: dalcinl at gmail.com */
+
+#ifndef MPI_COMPAT_H
+#define MPI_COMPAT_H
+
+#include <mpi.h>
+
+#if (MPI_VERSION < 3) && !defined(PyMPI_HAVE_MPI_Message)
+typedef void *PyMPI_MPI_Message;
+#define MPI_Message PyMPI_MPI_Message
+#endif
+
+#endif/*MPI_COMPAT_H*/
diff --git a/include/netCDF4.pxi b/include/netCDF4.pxi
index 3224cdb..3a8c837 100644
--- a/include/netCDF4.pxi
+++ b/include/netCDF4.pxi
@@ -696,6 +696,22 @@ IF HAS_NC_OPEN_MEM:
cdef extern from "netcdf_mem.h":
int nc_open_mem(const char *path, int mode, size_t size, void* memory, int *ncidp)
+IF HAS_NC_PAR:
+ cdef extern from "mpi-compat.h": pass
+ cdef extern from "netcdf_par.h":
+ ctypedef int MPI_Comm
+ ctypedef int MPI_Info
+ int nc_create_par(char *path, int cmode, MPI_Comm comm, MPI_Info info, int *ncidp);
+ int nc_open_par(char *path, int mode, MPI_Comm comm, MPI_Info info, int *ncidp);
+ int nc_var_par_access(int ncid, int varid, int par_access);
+ cdef enum:
+ NC_COLLECTIVE
+ NC_INDEPENDENT
+ cdef extern from "netcdf.h":
+ cdef enum:
+ NC_MPIIO
+ NC_PNETCDF
+
# taken from numpy.pxi in numpy 1.0rc2.
cdef extern from "numpy/arrayobject.h":
ctypedef int npy_intp
diff --git a/netCDF4/__init__.py b/netCDF4/__init__.py
index 65ba376..42e3113 100644
--- a/netCDF4/__init__.py
+++ b/netCDF4/__init__.py
@@ -5,6 +5,7 @@ from ._netCDF4 import *
from ._netCDF4 import __doc__, __pdoc__
from ._netCDF4 import (__version__, __netcdf4libversion__, __hdf5libversion__,
__has_rename_grp__, __has_nc_inq_path__,
- __has_nc_inq_format_extended__, __has_nc_open_mem__)
+ __has_nc_inq_format_extended__, __has_nc_open_mem__,
+ __has_cdf5_format__,__has_nc_par__)
__all__ =\
['Dataset','Variable','Dimension','Group','MFDataset','MFTime','CompoundType','VLType','date2num','num2date','date2index','stringtochar','chartostring','stringtoarr','getlibversion','EnumType']
diff --git a/netCDF4/_netCDF4.pyx b/netCDF4/_netCDF4.pyx
index 49b01c2..278d9a1 100644
--- a/netCDF4/_netCDF4.pyx
+++ b/netCDF4/_netCDF4.pyx
@@ -1,5 +1,5 @@
"""
-Version 1.3.0
+Version 1.3.1
-------------
- - -
@@ -38,7 +38,7 @@ Requires
- Python 2.7 or later (python 3 works too).
- [numpy array module](http://numpy.scipy.org), version 1.9.0 or later.
- - [Cython](http://cython.org), version 0.19 or later.
+ - [Cython](http://cython.org), version 0.21 or later.
- [setuptools](https://pypi.python.org/pypi/setuptools), version 18.0 or
later.
- The HDF5 C library version 1.8.4-patch1 or higher (1.8.x recommended)
@@ -60,6 +60,9 @@ Requires
If you want [OPeNDAP](http://opendap.org) support, add `--enable-dap`.
If you want HDF4 SD support, add `--enable-hdf4` and add
the location of the HDF4 headers and library to `$CPPFLAGS` and `$LDFLAGS`.
+ - for MPI parallel IO support, MPI-enabled versions of the HDF5 and netcdf
+ libraries are required, as is the [mpi4py](http://mpi4py.scipy.org) python
+ module.
Install
@@ -78,16 +81,16 @@ Install
- run `python setup.py build`, then `python setup.py install` (as root if
necessary).
- [`pip install`](https://pip.pypa.io/en/latest/reference/pip_install.html) can
- also be used, with library paths set with environment variables. To make
- this work, the `USE_SETUPCFG` environment variable must be used to tell
- setup.py not to use `setup.cfg`.
- For example, `USE_SETUPCFG=0 HDF5_INCDIR=/usr/include/hdf5/serial
- HDF5_LIBDIR=/usr/lib/x86_64-linux-gnu/hdf5/serial pip install` has been
- shown to work on an Ubuntu/Debian linux system. Similarly, environment variables
- (all capitalized) can be used to set the include and library paths for
- `hdf5`, `netCDF4`, `hdf4`, `szip`, `jpeg`, `curl` and `zlib`. If the
- libraries are installed in standard places (e.g. `/usr` or `/usr/local`),
- the environment variables do not need to be set.
+ also be used, with library paths set with environment variables. To make
+ this work, the `USE_SETUPCFG` environment variable must be used to tell
+ setup.py not to use `setup.cfg`.
+ For example, `USE_SETUPCFG=0 HDF5_INCDIR=/usr/include/hdf5/serial
+ HDF5_LIBDIR=/usr/lib/x86_64-linux-gnu/hdf5/serial pip install` has been
+ shown to work on an Ubuntu/Debian linux system. Similarly, environment variables
+ (all capitalized) can be used to set the include and library paths for
+ `hdf5`, `netCDF4`, `hdf4`, `szip`, `jpeg`, `curl` and `zlib`. If the
+ libraries are installed in standard places (e.g. `/usr` or `/usr/local`),
+ the environment variables do not need to be set.
- run the tests in the 'test' directory by running `python run_all.py`.
Tutorial
@@ -105,6 +108,7 @@ Tutorial
10. [Beyond homogeneous arrays of a fixed type - compound data types.](#section10)
11. [Variable-length (vlen) data types.](#section11)
12. [Enum data type.](#section12)
+13. [Parallel IO.](#section13)
## <div id='section1'>1) Creating/Opening/Closing a netCDF file.
@@ -893,7 +897,70 @@ specified names.
[0 2 4 -- 1]
>>> nc.close()
-All of the code in this tutorial is available in `examples/tutorial.py`,
+## <div id='section13'>13) Parallel IO.
+
+If MPI parallel enabled versions of netcdf and hdf5 are detected, and
+[mpi4py](https://mpi4py.scipy.org) is installed, netcdf4-python will
+be built with parallel IO capabilities enabled. To use parallel IO,
+your program must be running in an MPI environment using
+[mpi4py](https://mpi4py.scipy.org).
+
+ :::python
+ >>> from mpi4py import MPI
+ >>> import numpy as np
+ >>> from netCDF4 import Dataset
+ >>> rank = MPI.COMM_WORLD.rank # The process ID (integer 0-3 for 4-process run)
+
+To run an MPI-based parallel program like this, you must use `mpiexec` to launch several
+parallel instances of Python (for example, using `mpiexec -np 4 python mpi_example.py`).
+The parallel features of netcdf4-python are mostly transparent -
+when a new dataset is created or an existing dataset is opened,
+use the `parallel` keyword to enable parallel access.
+
+ :::python
+ >>> nc = Dataset('parallel_tst.nc','w',parallel=True)
+
+The optional `comm` keyword may be used to specify a particular
+MPI communicator (`MPI_COMM_WORLD` is used by default). Each process (or rank)
+can now write to the file indepedently. In this example the process rank is
+written to a different variable index on each task
+
+ :::python
+ >>> d = nc.createDimension('dim',4)
+ >>> v = nc.createVariable('var', np.int, 'dim')
+ >>> v[rank] = rank
+ >>> nc.close()
+
+ % ncdump parallel_test.nc
+ netcdf parallel_test {
+ dimensions:
+ dim = 4 ;
+ variables:
+ int64 var(dim) ;
+ data:
+
+ var = 0, 1, 2, 3 ;
+ }
+
+There are two types of parallel IO, independent (the default) and collective.
+Independent IO means that each process can do IO independently. It should not
+depend on or be affected by other processes. Collective IO is a way of doing
+IO defined in the MPI-IO standard; unlike independent IO, all processes must
+participate in doing IO. To toggle back and forth between
+the two types of IO, use the `netCDF4.Variable.set_collective`
+`netCDF4.Variable`method. All metadata
+operations (such as creation of groups, types, variables, dimensions, or attributes)
+are collective. There are a couple of important limitatons of parallel IO:
+
+ - If a variable has an unlimited dimension, appending data must be done in collective mode.
+ If the write is done in independent mode, the operation will fail with a
+ a generic "HDF Error".
+ - You cannot write compressed data in parallel (although
+ you can read it).
+ - You cannot use variable-length (VLEN) data types.
+
+All of the code in this tutorial is available in `examples/tutorial.py`, except
+the parallel IO example, which is in `examples/mpi_example.py`.
Unit tests are in the `test` directory.
**contact**: Jeffrey Whitaker <jeffrey.s.whitaker at noaa.gov>
@@ -936,7 +1003,7 @@ except ImportError:
# python3: zip is already python2's itertools.izip
pass
-__version__ = "1.3.0"
+__version__ = "1.3.1"
# Initialize numpy
import posixpath
@@ -952,6 +1019,16 @@ from libc.stdlib cimport malloc, free
import_array()
include "constants.pyx"
include "netCDF4.pxi"
+IF HAS_NC_PAR:
+ cimport mpi4py.MPI as MPI
+ from mpi4py.libmpi cimport MPI_Comm, MPI_Info, MPI_Comm_dup, MPI_Info_dup, \
+ MPI_Comm_free, MPI_Info_free, MPI_INFO_NULL,\
+ MPI_COMM_WORLD
+ ctypedef MPI.Comm Comm
+ ctypedef MPI.Info Info
+ELSE:
+ ctypedef object Comm
+ ctypedef object Info
# check for required version of netcdf-4 and hdf5.
@@ -977,8 +1054,9 @@ __hdf5libversion__ = _gethdf5libversion()
__has_rename_grp__ = HAS_RENAME_GRP
__has_nc_inq_path__ = HAS_NC_INQ_PATH
__has_nc_inq_format_extended__ = HAS_NC_INQ_FORMAT_EXTENDED
-__has_cdf5__ = HAS_CDF5_FORMAT
+__has_cdf5_format__ = HAS_CDF5_FORMAT
__has_nc_open_mem__ = HAS_NC_OPEN_MEM
+__has_nc_par__ = HAS_NC_PAR
_needsworkaround_issue485 = __netcdf4libversion__ < "4.4.0" or \
(__netcdf4libversion__.startswith("4.4.0") and \
"-development" in __netcdf4libversion__)
@@ -1550,10 +1628,14 @@ cdef _get_vars(group):
free(varids) # free pointer holding variable ids.
return variables
-cdef _ensure_nc_success(ierr, err_cls=RuntimeError):
+cdef _ensure_nc_success(ierr, err_cls=RuntimeError, filename=None):
# print netcdf error message, raise error.
if ierr != NC_NOERR:
- raise err_cls((<char *>nc_strerror(ierr)).decode('ascii'))
+ err_str = (<char *>nc_strerror(ierr)).decode('ascii')
+ if issubclass(err_cls, EnvironmentError):
+ raise err_cls(ierr, err_str, filename)
+ else:
+ raise err_cls(err_str)
# these are class attributes that
# only exist at the python level (not in the netCDF file).
@@ -1691,8 +1773,9 @@ references to the parent Dataset or Group.
the parent Dataset or Group."""
def __init__(self, filename, mode='r', clobber=True, format='NETCDF4',
- diskless=False, persist=False, keepweakref=False,
- memory=None, encoding=None, **kwargs):
+ diskless=False, persist=False, keepweakref=False,
+ memory=None, encoding=None, parallel=False,
+ Comm comm=None, Info info=None, **kwargs):
"""
**`__init__(self, filename, mode="r", clobber=True, diskless=False,
persist=False, keepweakref=False, format='NETCDF4')`**
@@ -1763,10 +1846,23 @@ references to the parent Dataset or Group.
**`encoding`**: encoding used to encode filename string into bytes.
Default is None (`sys.getdefaultfileencoding()` is used).
+
+ **`parallel`**: open for parallel access using MPI (requires mpi4py and
+ parallel-enabled netcdf-c and hdf5 libraries). Default is `False`. If
+ `True`, `comm` and `info` kwargs may also be specified.
+
+ **`comm`**: MPI_Comm object for parallel access. Default `None`, which
+ means MPI_COMM_WORLD will be used. Ignored if `parallel=False`.
+
+ **`info`**: MPI_Info object for parallel access. Default `None`, which
+ means MPI_INFO_NULL will be used. Ignored if `parallel=False`.
"""
cdef int grpid, ierr, numgrps, numdims, numvars
cdef char *path
cdef char namstring[NC_MAX_NAME+1]
+ IF HAS_NC_PAR:
+ cdef MPI_Comm mpicomm
+ cdef MPI_Info mpiinfo
memset(&self._buffer, 0, sizeof(self._buffer))
@@ -1784,11 +1880,32 @@ references to the parent Dataset or Group.
if memory is not None and (mode != 'r' or type(memory) != bytes):
raise ValueError('memory mode only works with \'r\' modes and must be `bytes`')
+ if parallel:
+ IF HAS_NC_PAR != 1:
+ msg='parallel mode requires MPI enabled netcdf-c'
+ raise ValueError(msg)
+ if format != 'NETCDF4':
+ msg='parallel mode only works with format=NETCDF4'
+ raise ValueError(msg)
+ if comm is not None:
+ mpicomm = comm.ob_mpi
+ else:
+ mpicomm = MPI_COMM_WORLD
+ if info is not None:
+ mpiinfo = info.ob_mpi
+ else:
+ mpiinfo = MPI_INFO_NULL
if mode == 'w':
_set_default_format(format=format)
if clobber:
- if diskless:
+ if parallel:
+ IF HAS_NC_PAR:
+ ierr = nc_create_par(path, NC_CLOBBER | NC_MPIIO, \
+ mpicomm, mpiinfo, &grpid)
+ ELSE:
+ pass
+ elif diskless:
if persist:
ierr = nc_create(path, NC_WRITE | NC_CLOBBER | NC_DISKLESS , &grpid)
else:
@@ -1796,7 +1913,13 @@ references to the parent Dataset or Group.
else:
ierr = nc_create(path, NC_CLOBBER, &grpid)
else:
- if diskless:
+ if parallel:
+ IF HAS_NC_PAR:
+ ierr = nc_create_par(path, NC_NOCLOBBER | NC_MPIIO, \
+ mpicomm, mpiinfo, &grpid)
+ ELSE:
+ pass
+ elif diskless:
if persist:
ierr = nc_create(path, NC_WRITE | NC_NOCLOBBER | NC_DISKLESS , &grpid)
else:
@@ -1822,23 +1945,49 @@ references to the parent Dataset or Group.
nc_open_mem method not enabled. To enable, install Cython, make sure you have
version 4.4.1 or higher of the netcdf C lib, and rebuild netcdf4-python."""
raise ValueError(msg)
+ elif parallel:
+ IF HAS_NC_PAR:
+ ierr = nc_open_par(path, NC_NOWRITE | NC_MPIIO, \
+ mpicomm, mpiinfo, &grpid)
+ ELSE:
+ pass
elif diskless:
ierr = nc_open(path, NC_NOWRITE | NC_DISKLESS, &grpid)
else:
ierr = nc_open(path, NC_NOWRITE, &grpid)
elif mode == 'r+' or mode == 'a':
- if diskless:
+ if parallel:
+ IF HAS_NC_PAR:
+ ierr = nc_open_par(path, NC_WRITE | NC_MPIIO, \
+ mpicomm, mpiinfo, &grpid)
+ ELSE:
+ pass
+ elif diskless:
ierr = nc_open(path, NC_WRITE | NC_DISKLESS, &grpid)
else:
ierr = nc_open(path, NC_WRITE, &grpid)
elif mode == 'as' or mode == 'r+s':
- if diskless:
+ if parallel:
+ # NC_SHARE ignored
+ IF HAS_NC_PAR:
+ ierr = nc_open_par(path, NC_WRITE | NC_MPIIO, \
+ mpicomm, mpiinfo, &grpid)
+ ELSE:
+ pass
+ elif diskless:
ierr = nc_open(path, NC_SHARE | NC_DISKLESS, &grpid)
else:
ierr = nc_open(path, NC_SHARE, &grpid)
elif mode == 'ws':
if clobber:
- if diskless:
+ if parallel:
+ # NC_SHARE ignored
+ IF HAS_NC_PAR:
+ ierr = nc_create_par(path, NC_CLOBBER | NC_MPIIO, \
+ mpicomm, mpiinfo, &grpid)
+ ELSE:
+ pass
+ elif diskless:
if persist:
ierr = nc_create(path, NC_WRITE | NC_SHARE | NC_CLOBBER | NC_DISKLESS , &grpid)
else:
@@ -1846,7 +1995,14 @@ references to the parent Dataset or Group.
else:
ierr = nc_create(path, NC_SHARE | NC_CLOBBER, &grpid)
else:
- if diskless:
+ if parallel:
+ # NC_SHARE ignored
+ IF HAS_NC_PAR:
+ ierr = nc_create_par(path, NC_NOCLOBBER | NC_MPIIO, \
+ mpicomm, mpiinfo, &grpid)
+ ELSE:
+ pass
+ elif diskless:
if persist:
ierr = nc_create(path, NC_WRITE | NC_SHARE | NC_NOCLOBBER | NC_DISKLESS , &grpid)
else:
@@ -1856,7 +2012,7 @@ references to the parent Dataset or Group.
else:
raise ValueError("mode must be 'w', 'r', 'a' or 'r+', got '%s'" % mode)
- _ensure_nc_success(ierr, IOError)
+ _ensure_nc_success(ierr, err_cls=IOError, filename=path)
# data model and file format attributes
self.data_model = _get_format(grpid)
@@ -3313,14 +3469,16 @@ behavior is similar to Fortran or Matlab, but different than numpy.
if grp.data_model != 'NETCDF4': grp._enddef()
_ensure_nc_success(ierr)
else:
- # cast fill_value to type of variable.
- # also make sure it is written in native byte order
- # (the same as the data)
- if self._isprimitive or self._isenum:
- fillval = numpy.array(fill_value, self.dtype)
- if not fillval.dtype.isnative: fillval.byteswap(True)
- _set_att(self._grp, self._varid, '_FillValue',\
- fillval, xtype=xtype)
+ if self._isprimitive or self._isenum or \
+ (self._isvlen and self.dtype == str):
+ if self._isvlen and self.dtype == str:
+ _set_att(self._grp, self._varid, '_FillValue',\
+ _tostr(fill_value), xtype=xtype, force_ncstring=True)
+ else:
+ fillval = numpy.array(fill_value, self.dtype)
+ if not fillval.dtype.isnative: fillval.byteswap(True)
+ _set_att(self._grp, self._varid, '_FillValue',\
+ fillval, xtype=xtype)
else:
raise AttributeError("cannot set _FillValue attribute for VLEN or compound variable")
if least_significant_digit is not None:
@@ -4320,10 +4478,7 @@ rightmost dimension of the variable.
The default value of `chartostring` is `True`
(automatic conversions are performed).
"""
- if chartostring:
- self.chartostring = True
- else:
- self.chartostring = False
+ self.chartostring = bool(chartostring)
def use_nc_get_vars(self,use_nc_get_vars):
"""
@@ -4334,11 +4489,8 @@ to retrieve strided variable slices. By default,
`nc_get_vars` not used since it slower than multiple calls
to the unstrided read routine `nc_get_vara` in most cases.
"""
- if not use_nc_get_vars:
- self._no_get_vars = True
- else:
- self._no_get_vars = False
-
+ self._no_get_vars = not bool(use_nc_get_vars)
+
def set_auto_maskandscale(self,maskandscale):
"""
**`set_auto_maskandscale(self,maskandscale)`**
@@ -4390,12 +4542,7 @@ data model does not have unsigned integer data types).
The default value of `maskandscale` is `True`
(automatic conversions are performed).
"""
- if maskandscale:
- self.scale = True
- self.mask = True
- else:
- self.scale = False
- self.mask = False
+ self.scale = self.mask = bool(maskandscale)
def set_auto_scale(self,scale):
"""
@@ -4434,11 +4581,8 @@ data model does not have unsigned integer data types).
The default value of `scale` is `True`
(automatic conversions are performed).
"""
- if scale:
- self.scale = True
- else:
- self.scale = False
-
+ self.scale = bool(scale)
+
def set_auto_mask(self,mask):
"""
**`set_auto_mask(self,mask)`**
@@ -4463,11 +4607,8 @@ set to missing_value.
The default value of `mask` is `True`
(automatic conversions are performed).
"""
- if mask:
- self.mask = True
- else:
- self.mask = False
-
+ self.mask = bool(mask)
+
def _put(self,ndarray data,start,count,stride):
"""Private method to put data into a netCDF variable"""
@@ -4746,6 +4887,25 @@ The default value of `mask` is `True`
else:
return data
+ def set_collective(self, value):
+ """
+**`set_collective(self,True_or_False)`**
+
+turn on or off collective parallel IO access. Ignored if file is not
+open for parallel access.
+ """
+ IF HAS_NC_PAR:
+ # set collective MPI IO mode on or off
+ if value:
+ ierr = nc_var_par_access(self._grpid, self._varid,
+ NC_COLLECTIVE)
+ else:
+ ierr = nc_var_par_access(self._grpid, self._varid,
+ NC_INDEPENDENT)
+ _ensure_nc_success(ierr)
+ ELSE:
+ pass # does nothing
+
def __reduce__(self):
# raise error is user tries to pickle a Variable object.
raise NotImplementedError('Variable is not picklable')
@@ -6081,7 +6241,7 @@ class _Variable(object):
class MFTime(_Variable):
"""
Class providing an interface to a MFDataset time Variable by imposing a unique common
-time unit to all files.
+time unit and/or calendar to all files.
Example usage (See `netCDF4.MFTime.__init__` for more details):
@@ -6113,17 +6273,22 @@ Example usage (See `netCDF4.MFTime.__init__` for more details):
32
"""
- def __init__(self, time, units=None):
+ def __init__(self, time, units=None, calendar=None):
"""
- **`__init__(self, time, units=None)`**
+ **`__init__(self, time, units=None, calendar=None)`**
Create a time Variable with units consistent across a multifile
dataset.
**`time`**: Time variable from a `netCDF4.MFDataset`.
- **`units`**: Time units, for example, `days since 1979-01-01`. If None, use
- the units from the master variable.
+ **`units`**: Time units, for example, `'days since 1979-01-01'`. If `None`,
+ use the units from the master variable.
+
+ **`calendar`**: Calendar overload to use across all files, for example,
+ `'standard'` or `'gregorian'`. If `None`, check that the calendar attribute
+ is present on each variable and values are unique across files raising a
+ `ValueError` otherwise.
"""
import datetime
self.__time = time
@@ -6132,14 +6297,27 @@ Example usage (See `netCDF4.MFTime.__init__` for more details):
for name, value in time.__dict__.items():
self.__dict__[name] = value
- # make sure calendar attribute present in all files.
- for t in self._recVar:
- if not hasattr(t,'calendar'):
- raise ValueError('MFTime requires that the time variable in all files have a calendar attribute')
+ # Make sure calendar attribute present in all files if no default calendar
+ # is provided. Also assert this value is the same across files.
+ if calendar is None:
+ calendars = [None] * len(self._recVar)
+ for idx, t in enumerate(self._recVar):
+ if not hasattr(t, 'calendar'):
+ msg = 'MFTime requires that the time variable in all files ' \
+ 'have a calendar attribute if no default calendar is provided.'
+ raise ValueError(msg)
+ else:
+ calendars[idx] = t.calendar
+ calendars = set(calendars)
+ if len(calendars) > 1:
+ msg = 'MFTime requires that the same time calendar is ' \
+ 'used by all files if no default calendar is provided.'
+ raise ValueError(msg)
+ else:
+ calendar = list(calendars)[0]
- # Check that calendar is the same in all files.
- if len(set([t.calendar for t in self._recVar])) > 1:
- raise ValueError('MFTime requires that the same time calendar is used by all files.')
+ # Set calendar using the default or the unique calendar value across all files.
+ self.calendar = calendar
# Override units if units is specified.
self.units = units or time.units
diff --git a/setup.cfg b/setup.cfg
index d4f522a..9b0094d 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -6,7 +6,7 @@
# Usually, nothing else is needed.
use_ncconfig=True
# path to nc-config script (use if not found in unix PATH).
-#ncconfig=/usr/local/bin/nc-config
+#ncconfig=/usr/local/bin/nc-config
[directories]
#
# If nc-config doesn't do the trick, you can specify the locations
@@ -46,3 +46,5 @@ use_ncconfig=True
# If the libraries and include files are installed in separate locations,
# use curl_libdir and curl_incdir.
#curl_dir = /usr/local
+# location of mpi.h (needed for parallel support)
+#mpi_incdir=/opt/local/include/mpich-mp
diff --git a/setup.py b/setup.py
index 119dfec..244e564 100644
--- a/setup.py
+++ b/setup.py
@@ -55,6 +55,7 @@ def check_api(inc_dirs):
has_nc_inq_format_extended = False
has_cdf5_format = False
has_nc_open_mem = False
+ has_nc_par = False
for d in inc_dirs:
try:
@@ -63,6 +64,7 @@ def check_api(inc_dirs):
continue
has_nc_open_mem = os.path.exists(os.path.join(d, 'netcdf_mem.h'))
+ has_nc_par = os.path.exists(os.path.join(d, 'netcdf_par.h'))
for line in f:
if line.startswith('nc_rename_grp'):
@@ -73,10 +75,17 @@ def check_api(inc_dirs):
has_nc_inq_format_extended = True
if line.startswith('#define NC_FORMAT_64BIT_DATA'):
has_cdf5_format = True
+
+ ncmetapath = os.path.join(d,'netcdf_meta.h')
+ if os.path.exists(ncmetapath):
+ has_cdf5 = False
+ for line in open(ncmetapath):
+ if line.startswith('#define NC_HAS_CDF5'):
+ has_cdf5 = True
break
return has_rename_grp, has_nc_inq_path, has_nc_inq_format_extended, \
- has_cdf5_format, has_nc_open_mem
+ has_cdf5_format, has_nc_open_mem, has_nc_par
def getnetcdfvers(libdirs):
@@ -139,6 +148,7 @@ jpeg_incdir = os.environ.get('JPEG_INCDIR')
curl_dir = os.environ.get('CURL_DIR')
curl_libdir = os.environ.get('CURL_LIBDIR')
curl_incdir = os.environ.get('CURL_INCDIR')
+mpi_incdir = os.environ.get('MPI_INCDIR')
USE_NCCONFIG = os.environ.get('USE_NCCONFIG')
if USE_NCCONFIG is not None:
@@ -233,6 +243,10 @@ if USE_SETUPCFG and os.path.exists(setup_cfg):
except:
pass
try:
+ mpi_incdir = config.get("directories","mpi_incdir")
+ except:
+ pass
+ try:
use_ncconfig = config.getboolean("options", "use_ncconfig")
except:
pass
@@ -442,7 +456,6 @@ if any('--' + opt in sys.argv for opt in Distribution.display_option_names +
else:
# append numpy include dir.
import numpy
-
inc_dirs.append(numpy.get_include())
# get netcdf library version.
@@ -455,16 +468,26 @@ else:
cmdclass = {}
netcdf4_src_root = osp.join('netCDF4', '_netCDF4')
netcdf4_src_c = netcdf4_src_root + '.c'
+netcdftime_src_root = osp.join('netcdftime', '_netcdftime')
+netcdftime_src_c = netcdftime_src_root + '.c'
if 'sdist' not in sys.argv[1:] and 'clean' not in sys.argv[1:]:
sys.stdout.write('using Cython to compile netCDF4.pyx...\n')
- # remove netCDF4.c file if it exists, so cython will recompile netCDF4.pyx.
+ # remove _netCDF4.c file if it exists, so cython will recompile _netCDF4.pyx.
# run for build *and* install (issue #263). Otherwise 'pip install' will
- # not regenerate netCDF4.c, even if the C lib supports the new features.
- if len(sys.argv) >= 2 and os.path.exists(netcdf4_src_c):
- os.remove(netcdf4_src_c)
+ # not regenerate _netCDF4.c, even if the C lib supports the new features.
+ if len(sys.argv) >= 2:
+ if os.path.exists(netcdf4_src_c):
+ os.remove(netcdf4_src_c)
+ # same for _netcdftime.c
+ if os.path.exists(netcdftime_src_c):
+ os.remove(netcdftime_src_c)
# this determines whether renameGroup and filepath methods will work.
has_rename_grp, has_nc_inq_path, has_nc_inq_format_extended, \
- has_cdf5_format, has_nc_open_mem = check_api(inc_dirs)
+ has_cdf5_format, has_nc_open_mem, has_nc_par = check_api(inc_dirs)
+ try:
+ import mpi4py
+ except ImportError:
+ has_nc_par = False
f = open(osp.join('include', 'constants.pyx'), 'w')
if has_rename_grp:
@@ -503,7 +526,21 @@ if 'sdist' not in sys.argv[1:] and 'clean' not in sys.argv[1:]:
sys.stdout.write('netcdf lib does not have cdf-5 format capability\n')
f.write('DEF HAS_CDF5_FORMAT = 0\n')
+ if has_nc_par:
+ sys.stdout.write('netcdf lib has netcdf4 parallel functions\n')
+ f.write('DEF HAS_NC_PAR = 1\n')
+ else:
+ sys.stdout.write('netcdf lib does not have netcdf4 parallel functions\n')
+ f.write('DEF HAS_NC_PAR = 0\n')
+
f.close()
+
+ if has_nc_par:
+ inc_dirs.append(mpi4py.get_include())
+ # mpi_incdir should not be needed if using nc-config
+ # (should be included in nc-config --cflags)
+ if mpi_incdir is not None: inc_dirs.append(mpi_incdir)
+
ext_modules = [Extension("netCDF4._netCDF4",
[netcdf4_src_root + '.pyx'],
libraries=libs,
@@ -511,14 +548,14 @@ if 'sdist' not in sys.argv[1:] and 'clean' not in sys.argv[1:]:
include_dirs=inc_dirs + ['include'],
runtime_library_dirs=runtime_lib_dirs),
Extension('netcdftime._netcdftime',
- ['netcdftime/_netcdftime.pyx'])]
+ [netcdftime_src_root + '.pyx'])]
else:
ext_modules = None
setup(name="netCDF4",
cmdclass=cmdclass,
- version="1.3.0",
- long_description="netCDF version 4 has many features not found in earlier versions of the library, such as hierarchical groups, zlib compression, multiple unlimited dimensions, and new data types. It is implemented on top of HDF5. This module implements most of the new features, and can read and write netCDF files compatible with older versions of the library. The API is modelled after Scientific.IO.NetCDF, and should be familiar to users of that module.\n\nThis project has a `S [...]
+ version="1.3.1",
+ long_description="netCDF version 4 has many features not found in earlier versions of the library, such as hierarchical groups, zlib compression, multiple unlimited dimensions, and new data types. It is implemented on top of HDF5. This module implements most of the new features, and can read and write netCDF files compatible with older versions of the library. The API is modelled after Scientific.IO.NetCDF, and should be familiar to users of that module.\n\nThis project is hoste [...]
author="Jeff Whitaker",
author_email="jeffrey.s.whitaker at noaa.gov",
url="http://github.com/Unidata/netcdf4-python",
diff --git a/test/run_all.py b/test/run_all.py
index 1cdefc7..c51879e 100755
--- a/test/run_all.py
+++ b/test/run_all.py
@@ -1,5 +1,6 @@
-import glob, os, sys, unittest, netCDF4
+import glob, os, sys, unittest
from netCDF4 import getlibversion,__hdf5libversion__,__netcdf4libversion__,__version__
+from netCDF4 import __has_cdf5_format__, __has_nc_inq_path__, __has_nc_par__
# can also just run
# python -m unittest discover . 'tst*py'
@@ -14,13 +15,13 @@ if python3:
else:
test_files.remove('tst_unicode3.py')
sys.stdout.write('not running tst_unicode3.py ...\n')
-if __netcdf4libversion__ < '4.2.1':
+if __netcdf4libversion__ < '4.2.1' or __has_nc_par__:
test_files.remove('tst_diskless.py')
sys.stdout.write('not running tst_diskless.py ...\n')
-if __netcdf4libversion__ < '4.1.2':
+if not __has_nc_inq_path__:
test_files.remove('tst_filepath.py')
sys.stdout.write('not running tst_filepath.py ...\n')
-if __netcdf4libversion__ < '4.4.0' or sys.maxsize < 2**32:
+if not __has_cdf5_format__:
test_files.remove('tst_cdf5.py')
sys.stdout.write('not running tst_cdf5.py ...\n')
@@ -41,12 +42,13 @@ def test(verbosity=1):
runner.run(testsuite)
if __name__ == '__main__':
- import numpy
+ import numpy, cython
sys.stdout.write('\n')
sys.stdout.write('netcdf4-python version: %s\n' % __version__)
sys.stdout.write('HDF5 lib version: %s\n' % __hdf5libversion__)
sys.stdout.write('netcdf lib version: %s\n' % __netcdf4libversion__)
sys.stdout.write('numpy version %s\n' % numpy.__version__)
+ sys.stdout.write('cython version %s\n' % cython.__version__)
runner = unittest.TextTestRunner(verbosity=1)
result = runner.run(testsuite)
if not result.wasSuccessful():
diff --git a/test/tst_filepath.py b/test/tst_filepath.py
index 79ef9a7..9db5957 100644
--- a/test/tst_filepath.py
+++ b/test/tst_filepath.py
@@ -3,6 +3,7 @@ import tempfile
import unittest
import netCDF4
+
class test_filepath(unittest.TestCase):
def setUp(self):
@@ -23,5 +24,11 @@ class test_filepath(unittest.TestCase):
nc.close()
shutil.rmtree(tmpdir)
+ def test_no_such_file_raises(self):
+ fname = 'not_a_nc_file.nc'
+ with self.assertRaisesRegexp(IOError, fname):
+ netCDF4.Dataset(fname, 'r')
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/test/tst_masked5.py b/test/tst_masked5.py
index 8b376bd..af49bf1 100755
--- a/test/tst_masked5.py
+++ b/test/tst_masked5.py
@@ -5,7 +5,7 @@ import tempfile
import numpy as np
from numpy import ma
from numpy.testing import assert_array_equal
-from netCDF4 import Dataset
+from netCDF4 import Dataset, __netcdf4libversion__
# Test use of vector of missing values.
@@ -23,9 +23,12 @@ class VectorMissingValues(unittest.TestCase):
f = Dataset(self.testfile, 'w')
d = f.createDimension('x',6)
v = f.createVariable('v', "i2", 'x')
+ # issue 730: set fill_value for vlen str vars
+ v2 = f.createVariable('v2',str,'x',fill_value=u'<missing>')
v.missing_value = self.missing_values
v[:] = self.v
+ v2[0]='first'
f.close()
@@ -41,6 +44,7 @@ class VectorMissingValues(unittest.TestCase):
f = Dataset(self.testfile)
v = f.variables["v"]
+ v2 = f.variables["v2"]
self.assertTrue(isinstance(v[:], ma.core.MaskedArray))
assert_array_equal(v[:], self.v_ma)
assert_array_equal(v[2],self.v[2]) # issue #624.
@@ -48,6 +52,14 @@ class VectorMissingValues(unittest.TestCase):
self.assertTrue(isinstance(v[:], np.ndarray))
assert_array_equal(v[:], self.v)
+ # issue 730
+ # this part fails with netcdf 4.1.3
+ # a bug in vlen strings?
+ if __netcdf4libversion__ >= '4.4.0':
+ assert (v2[0]==u'first')
+ assert (v2[1]==u'<missing>')
+
+
f.close()
diff --git a/test/tst_multifile.py b/test/tst_multifile.py
index 5b8d774..0bc3043 100644
--- a/test/tst_multifile.py
+++ b/test/tst_multifile.py
@@ -86,7 +86,9 @@ class NonuniformTimeTestCase(unittest.TestCase):
yr = 1979+nfile
time.units = 'days since %s-01-01' % yr
- time.calendar = 'standard'
+ # Do not set the calendar attribute on the created files to test calendar
+ # overload.
+ # time.calendar = 'standard'
x = f.createVariable('x','f',('time', 'y', 'z'))
x.units = 'potatoes per square mile'
@@ -106,20 +108,23 @@ class NonuniformTimeTestCase(unittest.TestCase):
def runTest(self):
+ # The test files have no calendar attribute on the time variable.
+ calendar = 'standard'
+
# Get the real dates
dates = []
for file in self.files:
f = Dataset(file)
t = f.variables['time']
- dates.extend(num2date(t[:], t.units, t.calendar))
+ dates.extend(num2date(t[:], t.units, calendar))
f.close()
# Compare with the MF dates
f = MFDataset(self.files,check=True)
t = f.variables['time']
- mfdates = num2date(t[:], t.units, t.calendar)
- T = MFTime(t)
+ T = MFTime(t, calendar=calendar)
+ assert_equal(T.calendar, calendar)
assert_equal(len(T), len(t))
assert_equal(T.shape, t.shape)
assert_equal(T.dimensions, t.dimensions)
@@ -128,5 +133,23 @@ class NonuniformTimeTestCase(unittest.TestCase):
assert_equal(date2index(datetime.datetime(1980, 1, 2), T), 366)
f.close()
+ # Test exception is raised when no calendar attribute is available on the
+ # time variable.
+ with MFDataset(self.files, check=True) as ds:
+ with self.assertRaises(ValueError):
+ MFTime(ds.variables['time'])
+
+ # Test exception is raised when the calendar attribute is different on the
+ # variables. First, add calendar attributes to file. Note this will modify
+ # the files inplace.
+ calendars = ['standard', 'gregorian']
+ for idx, f in enumerate(self.files):
+ with Dataset(f, 'a') as ds:
+ ds.variables['time'].calendar = calendars[idx]
+ with MFDataset(self.files, check=True) as ds:
+ with self.assertRaises(ValueError):
+ MFTime(ds.variables['time'])
+
+
if __name__ == '__main__':
unittest.main()
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/netcdf4-python.git
More information about the Pkg-grass-devel
mailing list