[Git][debian-gis-team/python-geopandas][master] 4 commits: New upstream version 0.14.1

Bas Couwenberg (@sebastic) gitlab at salsa.debian.org
Sat Nov 11 13:30:20 GMT 2023



Bas Couwenberg pushed to branch master at Debian GIS Project / python-geopandas


Commits:
847b7159 by Bas Couwenberg at 2023-11-11T14:20:23+01:00
New upstream version 0.14.1
- - - - -
95b08427 by Bas Couwenberg at 2023-11-11T14:20:46+01:00
Update upstream source from tag 'upstream/0.14.1'

Update to upstream version '0.14.1'
with Debian dir bc38bf027c7bf44ed6bd2aa5f743f663d491b94f
- - - - -
8413cfa7 by Bas Couwenberg at 2023-11-11T14:21:05+01:00
New upstream release.

- - - - -
4247f126 by Bas Couwenberg at 2023-11-11T14:25:45+01:00
Set distribution to unstable.

- - - - -


19 changed files:

- .github/workflows/tests.yaml
- CHANGELOG.md
- + ci/envs/312-latest-conda-forge.yaml
- ci/envs/39-latest-conda-forge_no_fiona.yaml
- debian/changelog
- doc/source/docs/user_guide/io.rst
- doc/source/gallery/plotting_basemap_background.ipynb
- doc/source/getting_started/install.rst
- geopandas/_compat.py
- geopandas/_version.py
- geopandas/explore.py
- + geopandas/io/_pyarrow_hotfix.py
- geopandas/io/arrow.py
- geopandas/io/file.py
- + geopandas/io/tests/data/arrow/test_data_v1.0.0.feather
- + geopandas/io/tests/data/arrow/test_data_v1.0.0.parquet
- geopandas/io/tests/test_file.py
- geopandas/tests/test_pandas_methods.py
- geopandas/tools/tests/test_sjoin.py


Changes:

=====================================
.github/workflows/tests.yaml
=====================================
@@ -44,6 +44,7 @@ jobs:
           - ci/envs/310-pd20-conda-forge.yaml
           - ci/envs/310-latest-conda-forge.yaml
           - ci/envs/311-latest-conda-forge.yaml
+          - ci/envs/312-latest-conda-forge.yaml
         include:
           - env: ci/envs/39-latest-conda-forge_no_fiona.yaml
             os: macos-latest


=====================================
CHANGELOG.md
=====================================
@@ -1,5 +1,11 @@
 # Changelog
 
+## Version 0.14.1 (Nov 11, 2023)
+
+- The Parquet and Feather IO functions now support the latest 1.0.0 version
+  of the GeoParquet specification (geoparquet.org) (#2663).
+- Fix `read_parquet` and `read_feather` for [CVE-2023-47248](https://www.cve.org/CVERecord?id=CVE-2023-47248>) (#3070).
+
 ## Version 0.14 (Sep 15, 2023)
 
 GeoPandas will use Shapely 2.0 by default instead of PyGEOS when both Shapely >= 2.0 and


=====================================
ci/envs/312-latest-conda-forge.yaml
=====================================
@@ -0,0 +1,35 @@
+name: test
+channels:
+  - conda-forge
+dependencies:
+  - python=3.12
+  # required
+  - pandas
+  - shapely
+  - fiona
+  - pyproj
+  - packaging
+  # testing
+  - pytest
+  - pytest-cov
+  - pytest-xdist
+  - fsspec
+  # optional
+  - pyogrio
+  # - matplotlib
+  - mapclassify
+  - folium
+  - xyzservices
+  - scipy
+  - geopy
+  # - pointpats
+  - geodatasets
+  # - SQLalchemy>=2
+  # - psycopg2
+  # - libspatialite
+  # - geoalchemy2
+  - pyarrow
+  - pip
+  - pip:
+    - matplotlib
+    - pointpats


=====================================
ci/envs/39-latest-conda-forge_no_fiona.yaml
=====================================
@@ -18,7 +18,7 @@ dependencies:
   # - fsspec  # to have one non-minimal build without fsspec
   # optional
   - rtree
-  - matplotlib
+  - matplotlib>=3.6
   - mapclassify
   - folium
   - xyzservices


=====================================
debian/changelog
=====================================
@@ -1,3 +1,10 @@
+python-geopandas (0.14.1-1) unstable; urgency=medium
+
+  * Team upload.
+  * New upstream release.
+
+ -- Bas Couwenberg <sebastic at debian.org>  Sat, 11 Nov 2023 14:25:33 +0100
+
 python-geopandas (0.14.0-1) unstable; urgency=medium
 
   * Team upload.


=====================================
doc/source/docs/user_guide/io.rst
=====================================
@@ -276,7 +276,7 @@ the spatial information.
 
 .. note::
 
-    This is tracking version 1.0.0-beta.1 of the GeoParquet specification at:
+    This is tracking version 1.0.0 of the GeoParquet specification at:
     https://github.com/opengeospatial/geoparquet.
 
     Previous versions are still supported as well. By default, the latest


=====================================
doc/source/gallery/plotting_basemap_background.ipynb
=====================================
@@ -180,7 +180,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "By default, contextily uses the Stamen Terrain style. We can specify a\n",
+    "By default, contextily uses the OpenStreetMap HOT style. We can specify a\n",
     "different style using ``cx.providers``:\n",
     "\n"
    ]
@@ -192,7 +192,7 @@
    "outputs": [],
    "source": [
     "ax = df_wm.plot(figsize=(10, 10), alpha=0.5, edgecolor=\"k\")\n",
-    "cx.add_basemap(ax, source=cx.providers.Stamen.TonerLite)\n",
+    "cx.add_basemap(ax, source=cx.providers.CartoDB.Positron)\n",
     "ax.set_axis_off()"
    ]
   },
@@ -220,8 +220,8 @@
    "outputs": [],
    "source": [
     "ax = df_wm.plot(figsize=(10, 10), alpha=0.5, edgecolor=\"k\")\n",
-    "cx.add_basemap(ax, source=cx.providers.Stamen.TonerLite)\n",
-    "cx.add_basemap(ax, source=cx.providers.Stamen.TonerLabels)"
+    "cx.add_basemap(ax, source=cx.providers.CartoDB.PositronNoLabels)\n",
+    "cx.add_basemap(ax, source=cx.providers.CartoDB.PositronOnlyLabels)"
    ]
   },
   {
@@ -239,8 +239,8 @@
    "outputs": [],
    "source": [
     "ax = df_wm.plot(figsize=(10, 10), alpha=0.5, edgecolor=\"k\")\n",
-    "cx.add_basemap(ax, source=cx.providers.Stamen.Watercolor, zoom=12)\n",
-    "cx.add_basemap(ax, source=cx.providers.Stamen.TonerLabels, zoom=10)"
+    "cx.add_basemap(ax, source=cx.providers.CartoDB.PositronNoLabels, zoom=12)\n",
+    "cx.add_basemap(ax, source=cx.providers.CartoDB.PositronOnlyLabels, zoom=10)"
    ]
   }
  ],


=====================================
doc/source/getting_started/install.rst
=====================================
@@ -92,16 +92,14 @@ as well::
     When using pip to install GeoPandas, you need to make sure that all dependencies are
     installed correctly.
 
-    - `fiona`_ provides binary wheels with the dependencies included for Mac and Linux,
-      but not for Windows. Alternatively, you can install `pyogrio`_ which does
-      have wheels for Windows.
-    - `pyproj`_, `rtree`_, and `shapely`_ provide binary wheels with dependencies included
-      for Mac, Linux, and Windows.
-
-    Depending on your platform, you might need to compile and install their
-    C dependencies manually. We refer to the individual packages for more
-    details on installing those.
-    Using conda (see above) avoids the need to compile the dependencies yourself.
+    Our main dependencies (`shapely`_, `pyproj`_, `fiona`_, `pyogrio`_, `rtree`_) provide binary
+    wheels with dependencies included for Mac, Linux, and Windows.
+
+    However, depending on your platform or Python version, there might be no
+    pre-compiled wheels available, and then you need to compile and install their C
+    dependencies manually. We refer to the individual packages for more details on
+    installing those. Using conda (see above) avoids the need to compile the
+    dependencies yourself.
 
 Installing from source
 ----------------------


=====================================
geopandas/_compat.py
=====================================
@@ -18,6 +18,7 @@ PANDAS_GE_14 = Version(pd.__version__) >= Version("1.4.0rc0")
 PANDAS_GE_15 = Version(pd.__version__) >= Version("1.5.0")
 PANDAS_GE_20 = Version(pd.__version__) >= Version("2.0.0")
 PANDAS_GE_21 = Version(pd.__version__) >= Version("2.1.0")
+PANDAS_GE_22 = Version(pd.__version__) >= Version("2.2.0.dev0")
 
 
 # -----------------------------------------------------------------------------


=====================================
geopandas/_version.py
=====================================
@@ -25,9 +25,9 @@ def get_keywords() -> Dict[str, str]:
     # setup.py/versioneer.py will grep for the variable names, so they must
     # each be defined on a line of their own. _version.py will just call
     # get_keywords().
-    git_refnames = " (HEAD -> main, tag: v0.14.0)"
-    git_full = "0eb2a5ecdc3b7b595e406c9f7bf1e6435ad39828"
-    git_date = "2023-09-15 10:24:28 +0200"
+    git_refnames = " (tag: v0.14.1, 0.14.x)"
+    git_full = "9a9f0974db087ce303b94bfbeabc8ea136be0914"
+    git_date = "2023-11-11 10:29:16 +0100"
     keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
     return keywords
 


=====================================
geopandas/explore.py
=====================================
@@ -93,8 +93,7 @@ def _explore(
         pass :class:`xyzservices.TileProvider` object or pass custom XYZ URL.
         The current list of built-in providers (when ``xyzservices`` is not available):
 
-        ``["OpenStreetMap", "Stamen Terrain", “Stamen Toner", “Stamen Watercolor"
-        "CartoDB positron", “CartoDB dark_matter"]``
+        ``["OpenStreetMap", "CartoDB positron", “CartoDB dark_matter"]``
 
         You can pass a custom tileset to Folium by passing a Leaflet-style URL
         to the tiles parameter: ``http://{s}.yourtiles.com/{z}/{x}/{y}.png``.
@@ -908,8 +907,7 @@ def _explore_geoseries(
         pass :class:`xyzservices.TileProvider` object or pass custom XYZ URL.
         The current list of built-in providers (when ``xyzservices`` is not available):
 
-        ``["OpenStreetMap", "Stamen Terrain", “Stamen Toner", “Stamen Watercolor"
-        "CartoDB positron", “CartoDB dark_matter"]``
+        ``["OpenStreetMap", "CartoDB positron", “CartoDB dark_matter"]``
 
         You can pass a custom tileset to Folium by passing a Leaflet-style URL
         to the tiles parameter: ``http://{s}.yourtiles.com/{z}/{x}/{y}.png``.


=====================================
geopandas/io/_pyarrow_hotfix.py
=====================================
@@ -0,0 +1,73 @@
+from packaging.version import Version
+
+import pyarrow
+
+
+_ERROR_MSG = """\
+Disallowed deserialization of 'arrow.py_extension_type':
+storage_type = {storage_type}
+serialized = {serialized}
+pickle disassembly:\n{pickle_disassembly}
+
+Reading of untrusted Parquet or Feather files with a PyExtensionType column
+allows arbitrary code execution.
+If you trust this file, you can enable reading the extension type by one of:
+
+- upgrading to pyarrow >= 14.0.1, and call `pa.PyExtensionType.set_auto_load(True)`
+- install pyarrow-hotfix (`pip install pyarrow-hotfix`) and disable it by running
+  `import pyarrow_hotfix; pyarrow_hotfix.uninstall()`
+
+We strongly recommend updating your Parquet/Feather files to use extension types
+derived from `pyarrow.ExtensionType` instead, and register this type explicitly.
+See https://arrow.apache.org/docs/dev/python/extending_types.html#defining-extension-types-user-defined-types
+for more details.
+"""
+
+
+def patch_pyarrow():
+    # starting from pyarrow 14.0.1, it has its own mechanism
+    if Version(pyarrow.__version__) >= Version("14.0.1"):
+        return
+
+    # if the user has pyarrow_hotfix (https://github.com/pitrou/pyarrow-hotfix)
+    # installed, use this instead (which also ensures it works if they had
+    # called `pyarrow_hotfix.uninstall()`)
+    try:
+        import pyarrow_hotfix  # noqa: F401
+    except ImportError:
+        pass
+    else:
+        return
+
+    # if the hotfix is already installed and enabled
+    if getattr(pyarrow, "_hotfix_installed", False):
+        return
+
+    class ForbiddenExtensionType(pyarrow.ExtensionType):
+        def __arrow_ext_serialize__(self):
+            return b""
+
+        @classmethod
+        def __arrow_ext_deserialize__(cls, storage_type, serialized):
+            import io
+            import pickletools
+
+            out = io.StringIO()
+            pickletools.dis(serialized, out)
+            raise RuntimeError(
+                _ERROR_MSG.format(
+                    storage_type=storage_type,
+                    serialized=serialized,
+                    pickle_disassembly=out.getvalue(),
+                )
+            )
+
+    pyarrow.unregister_extension_type("arrow.py_extension_type")
+    pyarrow.register_extension_type(
+        ForbiddenExtensionType(pyarrow.null(), "arrow.py_extension_type")
+    )
+
+    pyarrow._hotfix_installed = True
+
+
+patch_pyarrow()


=====================================
geopandas/io/arrow.py
=====================================
@@ -12,8 +12,8 @@ from geopandas import GeoDataFrame
 import geopandas
 from .file import _expand_user
 
-METADATA_VERSION = "1.0.0-beta.1"
-SUPPORTED_VERSIONS = ["0.1.0", "0.4.0", "1.0.0-beta.1"]
+METADATA_VERSION = "1.0.0"
+SUPPORTED_VERSIONS = ["0.1.0", "0.4.0", "1.0.0-beta.1", "1.0.0"]
 # reference: https://github.com/opengeospatial/geoparquet
 
 # Metadata structure:
@@ -74,7 +74,7 @@ def _create_metadata(df, schema_version=None):
     Parameters
     ----------
     df : GeoDataFrame
-    schema_version : {'0.1.0', '0.4.0', '1.0.0-beta.1', None}
+    schema_version : {'0.1.0', '0.4.0', '1.0.0-beta.1', '1.0.0', None}
         GeoParquet specification version; if not provided will default to
         latest supported version.
 
@@ -295,7 +295,7 @@ def _to_parquet(
 
     Requires 'pyarrow'.
 
-    This is tracking version 1.0.0-beta.1 of the GeoParquet specification at:
+    This is tracking version 1.0.0 of the GeoParquet specification at:
     https://github.com/opengeospatial/geoparquet. Writing older versions is
     supported using the `schema_version` keyword.
 
@@ -312,7 +312,7 @@ def _to_parquet(
         output except `RangeIndex` which is stored as metadata only.
     compression : {'snappy', 'gzip', 'brotli', None}, default 'snappy'
         Name of the compression to use. Use ``None`` for no compression.
-    schema_version : {'0.1.0', '0.4.0', '1.0.0-beta.1', None}
+    schema_version : {'0.1.0', '0.4.0', '1.0.0', None}
         GeoParquet specification version; if not provided will default to
         latest supported version.
     **kwargs
@@ -346,7 +346,7 @@ def _to_feather(df, path, index=None, compression=None, schema_version=None, **k
 
     Requires 'pyarrow' >= 0.17.
 
-    This is tracking version 1.0.0-beta.1 of the GeoParquet specification for
+    This is tracking version 1.0.0 of the GeoParquet specification for
     the metadata at: https://github.com/opengeospatial/geoparquet. Writing
     older versions is supported using the `schema_version` keyword.
 
@@ -364,7 +364,7 @@ def _to_feather(df, path, index=None, compression=None, schema_version=None, **k
     compression : {'zstd', 'lz4', 'uncompressed'}, optional
         Name of the compression to use. Use ``"uncompressed"`` for no
         compression. By default uses LZ4 if available, otherwise uncompressed.
-    schema_version : {'0.1.0', '0.4.0', '1.0.0-beta.1', None}
+    schema_version : {'0.1.0', '0.4.0', '1.0.0', None}
         GeoParquet specification version for the metadata; if not provided
         will default to latest supported version.
     kwargs
@@ -535,7 +535,7 @@ def _read_parquet(path, columns=None, storage_options=None, **kwargs):
       columns, the first available geometry column will be set as the geometry
       column of the returned GeoDataFrame.
 
-    Supports versions 0.1.0, 0.4.0 and 1.0.0-beta.1 of the GeoParquet
+    Supports versions 0.1.0, 0.4.0 and 1.0.0 of the GeoParquet
     specification at: https://github.com/opengeospatial/geoparquet
 
     If 'crs' key is not present in the GeoParquet metadata associated with the
@@ -587,6 +587,8 @@ def _read_parquet(path, columns=None, storage_options=None, **kwargs):
     parquet = import_optional_dependency(
         "pyarrow.parquet", extra="pyarrow is required for Parquet support."
     )
+    import geopandas.io._pyarrow_hotfix  # noqa: F401
+
     # TODO(https://github.com/pandas-dev/pandas/pull/41194): see if pandas
     # adds filesystem as a keyword and match that.
     filesystem = kwargs.pop("filesystem", None)
@@ -632,7 +634,7 @@ def _read_feather(path, columns=None, **kwargs):
       columns, the first available geometry column will be set as the geometry
       column of the returned GeoDataFrame.
 
-    Supports versions 0.1.0, 0.4.0 and 1.0.0-beta.1 of the GeoParquet
+    Supports versions 0.1.0, 0.4.0 and 1.0.0 of the GeoParquet
     specification at: https://github.com/opengeospatial/geoparquet
 
     If 'crs' key is not present in the Feather metadata associated with the
@@ -675,6 +677,7 @@ def _read_feather(path, columns=None, **kwargs):
     )
     # TODO move this into `import_optional_dependency`
     import pyarrow
+    import geopandas.io._pyarrow_hotfix  # noqa: F401
 
     if Version(pyarrow.__version__) < Version("0.17.0"):
         raise ImportError("pyarrow >= 0.17 required for Feather support")


=====================================
geopandas/io/file.py
=====================================
@@ -417,6 +417,10 @@ def _read_file_pyogrio(path_or_bytes, bbox=None, mask=None, rows=None, **kwargs)
             kwargs["max_features"] = rows
         elif isinstance(rows, slice):
             if rows.start is not None:
+                if rows.start < 0:
+                    raise ValueError(
+                        "Negative slice start not supported with the 'pyogrio' engine."
+                    )
                 kwargs["skip_features"] = rows.start
             if rows.stop is not None:
                 kwargs["max_features"] = rows.stop - (rows.start or 0)


=====================================
geopandas/io/tests/data/arrow/test_data_v1.0.0.feather
=====================================
Binary files /dev/null and b/geopandas/io/tests/data/arrow/test_data_v1.0.0.feather differ


=====================================
geopandas/io/tests/data/arrow/test_data_v1.0.0.parquet
=====================================
Binary files /dev/null and b/geopandas/io/tests/data/arrow/test_data_v1.0.0.parquet differ


=====================================
geopandas/io/tests/test_file.py
=====================================
@@ -23,8 +23,11 @@ from geopandas.tests.util import PACKAGE_DIR, validate_boro_df
 
 try:
     import pyogrio
+
+    PYOGRIO_GE_07 = Version(pyogrio.__version__) > Version("0.6.0")
 except ImportError:
     pyogrio = False
+    PYOGRIO_GE_07 = False
 
 
 try:
@@ -733,7 +736,7 @@ def test_read_file_filtered__rows_bbox(df_nybb, engine):
         1047224.3104931959,
         244317.30894023244,
     )
-    if engine == "pyogrio":
+    if engine == "pyogrio" and not PYOGRIO_GE_07:
         with pytest.raises(ValueError, match="'skip_features' must be between 0 and 1"):
             # combination bbox and rows (rows slice applied after bbox filtering!)
             filtered_df = read_file(
@@ -748,7 +751,10 @@ def test_read_file_filtered__rows_bbox(df_nybb, engine):
 
     if engine == "pyogrio":
         # TODO: support negative rows in pyogrio
-        with pytest.raises(ValueError, match="'skip_features' must be between 0 and 1"):
+        with pytest.raises(
+            ValueError,
+            match="'skip_features' must be between 0 and 1|Negative slice start",
+        ):
             filtered_df = read_file(
                 nybb_filename, bbox=bbox, rows=slice(-1, None), engine=engine
             )


=====================================
geopandas/tests/test_pandas_methods.py
=====================================
@@ -251,7 +251,10 @@ def test_astype(s, df):
     assert s.astype(str)[0] == "POINT (0 0)"
 
     res = s.astype(object)
-    if not Version(pd.__version__) == Version("2.1.0"):
+    if not (
+        (Version(pd.__version__) == Version("2.1.0"))
+        or (Version(pd.__version__) == Version("2.1.1"))
+    ):
         # https://github.com/geopandas/geopandas/issues/2948 - bug in pandas 2.1.0
         assert isinstance(res, pd.Series) and not isinstance(res, GeoSeries)
         assert res.dtype == object
@@ -682,12 +685,20 @@ def test_groupby_metadata(crs):
         lambda x: geopandas.sjoin(x, x[["geometry", "value1"]], how="inner")
     )
 
+    if compat.PANDAS_GE_22:
+        # merge sort behaviour changed in pandas #54611
+        take_indices = [0, 0, 2, 2, 1]
+        value_right = [0, 2, 0, 2, 1]
+    else:
+        take_indices = [0, 2, 0, 2, 1]
+        value_right = [0, 0, 2, 2, 1]
+
     expected = (
-        df.take([0, 2, 0, 2, 1])
+        df.take(take_indices)
         .set_index("value2", drop=False, append=True)
         .swaplevel()
         .rename(columns={"value1": "value1_left"})
-        .assign(value1_right=[0, 0, 2, 2, 1])
+        .assign(value1_right=value_right)
     )
     assert_geodataframe_equal(res.drop(columns=["index_right"]), expected)
 


=====================================
geopandas/tools/tests/test_sjoin.py
=====================================
@@ -927,6 +927,10 @@ class TestNearest:
         result5["index_right"] = result5["index_right"].astype("int64")
         assert_geodataframe_equal(result5, result4, check_like=True)
 
+    expected_index_uncapped = (
+        [1, 3, 3, 1, 2] if compat.PANDAS_GE_22 else [1, 1, 3, 3, 2]
+    )
+
     @pytest.mark.skipif(
         not (compat.USE_SHAPELY_20),
         reason=(
@@ -935,7 +939,7 @@ class TestNearest:
         ),
     )
     @pytest.mark.parametrize(
-        "max_distance,expected", [(None, [1, 1, 3, 3, 2]), (1.1, [3, 3, 1, 2])]
+        "max_distance,expected", [(None, expected_index_uncapped), (1.1, [3, 3, 1, 2])]
     )
     def test_sjoin_nearest_exclusive(self, max_distance, expected):
         geoms = shapely.points(np.arange(3), np.arange(3))



View it on GitLab: https://salsa.debian.org/debian-gis-team/python-geopandas/-/compare/a3a040b5a0ddc20b232cff055dc1aeabeb1238dd...4247f126aa19f1e3bb647728b98b7f45b1443eca

-- 
View it on GitLab: https://salsa.debian.org/debian-gis-team/python-geopandas/-/compare/a3a040b5a0ddc20b232cff055dc1aeabeb1238dd...4247f126aa19f1e3bb647728b98b7f45b1443eca
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20231111/9421168b/attachment-0001.htm>


More information about the Pkg-grass-devel mailing list