[Python-modules-commits] [python-elasticsearch] 05/11: New upstream version 5.4.0
Sophie Brun
sbrun-guest at moszumanska.debian.org
Tue Sep 26 08:03:38 UTC 2017
This is an automated email from the git hooks/post-receive script.
sbrun-guest pushed a commit to branch debian/master
in repository python-elasticsearch.
commit c4b0a46a40974ec556f74ea5c3a12a328f8d4492
Author: Sophie Brun <sophie at freexian.com>
Date: Mon Sep 25 15:11:01 2017 +0200
New upstream version 5.4.0
---
.gitignore | 1 +
.gitmodules | 3 -
.travis.yml | 20 +-
AUTHORS | 13 +
CONTRIBUTING.md | 4 +-
Changelog.rst | 102 +-
README | 156 ++-
README.rst | 114 +--
docs/api.rst | 26 +-
docs/conf.py | 8 +-
docs/connection.rst | 2 +-
docs/exceptions.rst | 1 -
docs/helpers.rst | 81 ++
docs/index.rst | 122 ++-
docs/transports.rst | 31 +-
elasticsearch/__init__.py | 4 +-
elasticsearch/client/__init__.py | 1136 ++++++++++++++--------
elasticsearch/client/cat.py | 316 ++++--
elasticsearch/client/cluster.py | 125 ++-
elasticsearch/client/indices.py | 906 ++++++++---------
elasticsearch/client/ingest.py | 61 ++
elasticsearch/client/nodes.py | 81 +-
elasticsearch/client/snapshot.py | 83 +-
elasticsearch/client/tasks.py | 61 ++
elasticsearch/client/utils.py | 12 +-
elasticsearch/compat.py | 4 +-
elasticsearch/connection/__init__.py | 2 -
elasticsearch/connection/base.py | 76 +-
elasticsearch/connection/esthrift/Rest-remote | 88 --
elasticsearch/connection/esthrift/Rest.py | 223 -----
elasticsearch/connection/esthrift/__init__.py | 1 -
elasticsearch/connection/esthrift/constants.py | 11 -
elasticsearch/connection/esthrift/ttypes.py | 412 --------
elasticsearch/connection/http_requests.py | 54 +-
elasticsearch/connection/http_urllib3.py | 88 +-
elasticsearch/connection/memcached.py | 83 --
elasticsearch/connection/pooling.py | 13 +
elasticsearch/connection/thrift.py | 99 --
elasticsearch/connection_pool.py | 23 +-
elasticsearch/exceptions.py | 8 +-
elasticsearch/helpers/__init__.py | 338 ++++---
elasticsearch/helpers/test.py | 7 +-
elasticsearch/serializer.py | 5 +-
elasticsearch/transport.py | 98 +-
example/load.py | 148 ++-
example/queries.py | 69 +-
setup.py | 15 +-
start_elasticsearch.sh | 18 +
test_elasticsearch/run_tests.py | 2 +-
test_elasticsearch/test_client/__init__.py | 21 +-
test_elasticsearch/test_connection.py | 68 +-
test_elasticsearch/test_helpers.py | 42 +
test_elasticsearch/test_serializer.py | 4 +
test_elasticsearch/test_server/__init__.py | 17 +-
test_elasticsearch/test_server/test_client.py | 2 +-
test_elasticsearch/test_server/test_common.py | 55 +-
test_elasticsearch/test_server/test_helpers.py | 82 +-
test_elasticsearch/test_server/test_memcached.py | 42 -
test_elasticsearch/test_transport.py | 66 +-
59 files changed, 2997 insertions(+), 2756 deletions(-)
diff --git a/.gitignore b/.gitignore
index d00f577..3e8176d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
+.eggs/*
.*.swp
*~
*.py[co]
diff --git a/.gitmodules b/.gitmodules
deleted file mode 100644
index 49a3cf0..0000000
--- a/.gitmodules
+++ /dev/null
@@ -1,3 +0,0 @@
-[submodule "rest-api-spec"]
- path = rest-api-spec
- url = https://github.com/elasticsearch/elasticsearch.git
diff --git a/.travis.yml b/.travis.yml
index 5a00ffb..e1e1885 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,20 +3,30 @@ language: python
python:
- "2.6"
- "2.7"
- - "3.2"
- "3.3"
- "3.4"
- - "pypy"
+ - "3.5"
+ - "3.6"
+
+addons:
+ apt:
+ packages:
+ - oracle-java8-installer
env:
# different connection classes to test
- TEST_ES_CONNECTION=Urllib3HttpConnection
- TEST_ES_CONNECTION=RequestsHttpConnection
+before_install:
+ - sudo update-java-alternatives -s java-8-oracle
+ - export JAVA_HOME=/usr/lib/jvm/java-8-oracle/jre
+ - java -version
+
install:
- - mkdir /tmp/elasticsearch
- - wget -O - http://s3-eu-west-1.amazonaws.com/build-eu.elasticsearch.org/origin/master/nightly/JDK7/elasticsearch-latest-SNAPSHOT.tar.gz | tar xz --directory=/tmp/elasticsearch --strip-components=1
- - /tmp/elasticsearch/bin/elasticsearch -d --path.data /tmp --discovery.zen.ping.multicast.enabled false --script.inline on --script.indexed on
+ - curl -L -o /tmp/es-snap.zip https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.3.0.zip
+ - unzip /tmp/es-snap.zip -d /tmp/
+ - /tmp/elasticsearch-*/bin/elasticsearch -E script.inline=true -E path.repo=/tmp -E repositories.url.allowed_urls='http://*' -E node.attr.testattr=test -d
- git clone https://github.com/elastic/elasticsearch.git ../elasticsearch
- pip install .
diff --git a/AUTHORS b/AUTHORS
index cab058e..ed10c7f 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -43,3 +43,16 @@ Russell Savage <russell.savage at elastic.co>
Georges Toth <georges.toth at govcert.etat.lu>
Malthe Borch <mborch at gmail.com>
Jim Kelly <pthread1981 at gmail.com>
+Xiuming Chen <cc at cxm.cc>
+David Szotten <davidszotten at gmail.com>
+Jorgen Jorgensen <jayjaypy at barskdata.net>
+Jason Veatch <jtveatch at gmail.com>
+Daniel <dmg.lab at outlook.com>
+Emil Hessman <emil at hessman.se>
+Dmitry Sadovnychyi <dyriax at gmail.com>
+Chris Earle <chris.earle at elastic.co>
+Tomas Mozes <tmozes at sygic.com>
+Alexei Peters <apeters at fargeo.com>
+Michael Gibson <michael.x.gibson at oracle.com>
+Darryl Ring <darryl at darrylring.com>
+Will McGinnis <will at predikto.com>
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index c044c6e..cba0fed 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -23,7 +23,7 @@ The process for contributing to any of the Elasticsearch repositories is similar
python setup.py test
````
- See the README file in `test_elasticsearch` dirctory for more information on
+ See the README file in `test_elasticsearch` directory for more information on
running the test suite.
3. Rebase your changes.
@@ -37,7 +37,7 @@ The process for contributing to any of the Elasticsearch repositories is similar
place, eg “Closes #123″. Please consider adding or modifying tests related to
your changes.
-Then sit back and wait. There will probably be discussion about the pull
+Then sit back and wait. There will probably be a discussion about the pull
request and, if any changes are needed, we would love to work with you to get
your pull request merged into elasticsearch-py.
diff --git a/Changelog.rst b/Changelog.rst
index 3a9b594..1ccf774 100644
--- a/Changelog.rst
+++ b/Changelog.rst
@@ -3,10 +3,108 @@
Changelog
=========
+5.4.0 (2017-05-18)
+------------------
+
+ * ``bulk`` helpers now extract ``pipeline`` parameter from the action
+ dictionary.
+
+5.3.0 (2017-03-30)
+------------------
+
+Compatibility with elasticsearch 5.3
+
+5.2.0 (2017-02-12)
+------------------
+
+The client now automatically sends ``Content-Type`` http header set to
+``application/json``. If you are explicitly passing in other encoding than
+``json`` you need to set the header manually.
+
+5.1.0 (2017-01-11)
+------------------
+
+ * Fixed sniffing
+
+5.0.1 (2016-11-02)
+------------------
+
+Fixed performance regression in ``scan`` helper
+
+5.0.0 (2016-10-19)
+------------------
+
+Version compatible with elasticsearch 5.0
+
+ * when using SSL certificate validation is now on by default. Install
+ ``certifi`` or supply root certificate bundle.
+ * ``elasticsearch.trace`` logger now also logs failed requests, signature of
+ internal logging method ``log_request_fail`` has changed, all custom
+ connection classes need to be updated
+ * added ``headers`` arg to connections to support custom http headers
+ * passing in a keyword parameter with ``None`` as value will cause that param
+ to be ignored
+
+2.4.0 (2016-08-17)
+------------------
+
+ * ``ping`` now ignores all ``TransportError`` exceptions and just returns
+ ``False``
+ * expose ``scroll_id`` on ``ScanError``
+ * increase default size for ``scan`` helper to 1000
+
+Internal:
+
+ * changed ``Transport.perform_request`` to just return the body, not status as well.
+
+2.3.0 (2016-02-29)
+------------------
+
+ * added ``client_key`` argument to configure client certificates
+ * debug logging now includes response body even for failed requests
+
+2.2.0 (2016-01-05)
+------------------
+
+Due to change in json encoding the client will no longer mask issues with
+encoding - if you work with non-ascii data in python 2 you must use the
+``unicode`` type or have proper encoding set in your environment.
+
+ * adding additional options for ssh - ``ssl_assert_hostname`` and
+ ``ssl_assert_fingerprint`` to the default connection class
+ * fix sniffing
+
+2.1.0 (2015-10-19)
+------------------
+
+ * move multiprocessing import inside parallel bulk for Google App Engine
+
+2.0.0 (2015-10-14)
+------------------
+
+ * Elasticsearch 2.0 compatibility release
+
+1.8.0 (2015-10-14)
+------------------
+
+ * removed thrift and memcached connections, if you wish to continue using
+ those, extract the classes and use them separately.
+ * added a new, parallel version of the bulk helper using thread pools
+ * In helpers, removed ``bulk_index`` as an alias for ``bulk``. Use ``bulk``
+ instead.
+
+1.7.0 (2015-09-21)
+------------------
+
+ * elasticsearch 2.0 compatibility
+ * thrift now deprecated, to be removed in future version
+ * make sure urllib3 always uses keep-alive
+
1.6.0 (2015-06-10)
------------------
* Add ``indices.flush_synced`` API
+ * ``helpers.reindex`` now supports reindexing parent/child documents
1.5.0 (2015-05-18)
------------------
@@ -88,7 +186,7 @@ compatible with 0.90 elasticsearch.
backwards compatibility, to be removed in future versions)
* Added ``helpers.streaming_bulk`` to consume an iterator and yield results per
operation
- * ``helpers.bulk`` and ``helpers.streaming_bulk`` are no longer limitted to just
+ * ``helpers.bulk`` and ``helpers.streaming_bulk`` are no longer limited to just
index operations.
* unicode body (for ``incices.analyze`` for example) is now handled correctly
* changed ``perform_request`` on ``Connection`` classes to return headers as well.
@@ -109,7 +207,7 @@ compatible with 0.90 elasticsearch.
0.4.2 (2013-10-08)
------------------
- * ``ignore`` param acceted by all APIs
+ * ``ignore`` param accepted by all APIs
* Fixes to ``helpers.bulk_index``
0.4.1 (2013-09-24)
diff --git a/README b/README
deleted file mode 120000
index 92cacd2..0000000
--- a/README
+++ /dev/null
@@ -1 +0,0 @@
-README.rst
\ No newline at end of file
diff --git a/README b/README
new file mode 100644
index 0000000..ea522f1
--- /dev/null
+++ b/README
@@ -0,0 +1,155 @@
+Python Elasticsearch Client
+===========================
+
+Official low-level client for Elasticsearch. Its goal is to provide common
+ground for all Elasticsearch-related code in Python; because of this it tries
+to be opinion-free and very extendable.
+
+For a more high level client library with more limited scope, have a look at
+`elasticsearch-dsl`_ - a more pythonic library sitting on top of
+``elasticsearch-py``.
+
+It provides a more convenient and idiomatic way to write and manipulate
+`queries`_. It stays close to the Elasticsearch JSON DSL, mirroring its
+terminology and structure while exposing the whole range of the DSL from Python
+either directly using defined classes or a queryset-like expressions.
+
+It also provides an optional `persistence layer`_ for working with documents as
+Python objects in an ORM-like fashion: defining mappings, retrieving and saving
+documents, wrapping the document data in user-defined classes.
+
+.. _elasticsearch-dsl: https://elasticsearch-dsl.readthedocs.io/
+.. _queries: https://elasticsearch-dsl.readthedocs.io/en/latest/search_dsl.html
+.. _persistence layer: https://elasticsearch-dsl.readthedocs.io/en/latest/persistence.html#doctype
+
+Compatibility
+-------------
+
+The library is compatible with all Elasticsearch versions since ``0.90.x`` but you
+**have to use a matching major version**:
+
+For **Elasticsearch 5.0** and later, use the major version 5 (``5.x.y``) of the
+library.
+
+For **Elasticsearch 2.0** and later, use the major version 2 (``2.x.y``) of the
+library.
+
+For **Elasticsearch 1.0** and later, use the major version 1 (``1.x.y``) of the
+library.
+
+For **Elasticsearch 0.90.x**, use a version from ``0.4.x`` releases of the
+library.
+
+The recommended way to set your requirements in your `setup.py` or
+`requirements.txt` is::
+
+ # Elasticsearch 5.x
+ elasticsearch>=5.0.0,<6.0.0
+
+ # Elasticsearch 2.x
+ elasticsearch>=2.0.0,<3.0.0
+
+ # Elasticsearch 1.x
+ elasticsearch>=1.0.0,<2.0.0
+
+ # Elasticsearch 0.90.x
+ elasticsearch<1.0.0
+
+The development is happening on ``master`` and ``2.x`` branches respectively.
+
+Installation
+------------
+
+Install the ``elasticsearch`` package with `pip
+<https://pypi.python.org/pypi/elasticsearch>`_::
+
+ pip install elasticsearch
+
+
+Run Elasticsearch in a Container
+--------------------------------
+
+To run elasticsearch in a container, optionally set the `ES_VERSION` environment evariable to either 5.4, 5.3 or 2.4. `ES_VERSION` is defaulted to `latest`.
+Then run ./start_elasticsearch.sh::
+
+ export ES_VERSION=5.4
+ ./start_elasticsearch.sh
+
+
+This will run a version fo Elastic Search in a Docker container suitable for running the tests. To check that elasticearch is running
+first wait for a `healthy` status in `docker ps`::
+
+ $ docker ps
+ CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
+ 955e57564e53 7d2ad83f8446 "/docker-entrypoin..." 6 minutes ago Up 6 minutes (healthy) 0.0.0.0:9200->9200/tcp, 9300/tcp trusting_brattain
+
+Then you can navigate to `locahost:9200` in your browser.
+
+
+Example use
+-----------
+
+Simple use-case::
+
+ >>> from datetime import datetime
+ >>> from elasticsearch import Elasticsearch
+
+ # by default we connect to localhost:9200
+ >>> es = Elasticsearch()
+
+ # create an index in elasticsearch, ignore status code 400 (index already exists)
+ >>> es.indices.create(index='my-index', ignore=400)
+ {u'acknowledged': True}
+
+ # datetimes will be serialized
+ >>> es.index(index="my-index", doc_type="test-type", id=42, body={"any": "data", "timestamp": datetime.now()})
+ {u'_id': u'42', u'_index': u'my-index', u'_type': u'test-type', u'_version': 1, u'ok': True}
+
+ # but not deserialized
+ >>> es.get(index="my-index", doc_type="test-type", id=42)['_source']
+ {u'any': u'data', u'timestamp': u'2013-05-12T19:45:31.804229'}
+
+`Full documentation`_.
+
+.. _Full documentation: https://elasticsearch-py.readthedocs.io/
+
+
+Features
+--------
+
+The client's features include:
+
+ * translating basic Python data types to and from json (datetimes are not
+ decoded for performance reasons)
+ * configurable automatic discovery of cluster nodes
+ * persistent connections
+ * load balancing (with pluggable selection strategy) across all available nodes
+ * failed connection penalization (time based - failed connections won't be
+ retried until a timeout is reached)
+ * support for ssl and http authentication
+ * thread safety
+ * pluggable architecture
+
+
+License
+-------
+
+Copyright 2015 Elasticsearch
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Build status
+------------
+
+.. image:: https://secure.travis-ci.org/elastic/elasticsearch-py.png
+ :target: http://travis-ci.org/#!/elastic/elasticsearch-py
diff --git a/README.rst b/README.rst
deleted file mode 100644
index 76656ff..0000000
--- a/README.rst
+++ /dev/null
@@ -1,113 +0,0 @@
-Python Elasticsearch Client
-===========================
-
-Official low-level client for Elasticsearch. Its goal is to provide common
-ground for all Elasticsearch-related code in Python; because of this it tries
-to be opinion-free and very extendable.
-
-For a more high level client library with more limited scope, have a look at
-`elasticsearch-dsl`_ - it is a more pythonic library sitting on top of
-``elasticsearch-py``.
-
-.. _elasticsearch-dsl: http://elasticsearch-dsl.rtfd.org/
-
-Compatibility
--------------
-
-The library is compatible with both Elasticsearch 1.x and 0.90.x but you
-**have to use a matching version**.
-
-For **Elasticsearch 1.0** and later, use the major version 1 (``1.x.y``) of the
-library.
-
-For **Elasticsearch 0.90.x**, use a version from ``0.4.x`` releases of the
-library.
-
-The recommended way to set your requirements in your `setup.py` or
-`requirements.txt` is::
-
- # Elasticsearch 1.0
- elasticsearch>=1.0.0,<2.0.0
-
- # Elasticsearch 0.90
- elasticsearch<1.0.0
-
-The development is happening on ``master`` and ``0.4`` branches, respectively.
-
-
-Installation
-------------
-
-Install the ``elasticsearch`` package with `pip
-<https://pypi.python.org/pypi/elasticsearch>`_::
-
- pip install elasticsearch
-
-
-Example use
------------
-
-Simple use-case::
-
- >>> from datetime import datetime
- >>> from elasticsearch import Elasticsearch
-
- # by default we connect to localhost:9200
- >>> es = Elasticsearch()
-
- # create an index in elasticsearch, ignore status code 400 (index already exists)
- >>> es.indices.create(index='my-index', ignore=400)
- {u'acknowledged': True}
-
- # datetimes will be serialized
- >>> es.index(index="my-index", doc_type="test-type", id=42, body={"any": "data", "timestamp": datetime.now()})
- {u'_id': u'42', u'_index': u'my-index', u'_type': u'test-type', u'_version': 1, u'ok': True}
-
- # but not deserialized
- >>> es.get(index="my-index", doc_type="test-type", id=42)['_source']
- {u'any': u'data', u'timestamp': u'2013-05-12T19:45:31.804229'}
-
-`Full documentation`_.
-
-.. _Full documentation: http://elasticsearch-py.rtfd.org/
-
-
-Features
---------
-
-The client's features include:
-
- * translating basic Python data types to and from json (datetimes are not
- decoded for performance reasons)
- * configurable automatic discovery of cluster nodes
- * persistent connections
- * load balancing (with pluggable selection strategy) across all available nodes
- * failed connection penalization (time based - failed connections won't be
- retried until a timeout is reached)
- * support for ssl and http authentication
- * thread safety
- * pluggable architecture
-
-
-License
--------
-
-Copyright 2015 Elasticsearch
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Build status
-------------
-
-.. image:: https://secure.travis-ci.org/elastic/elasticsearch-py.png
- :target: http://travis-ci.org/#!/elastic/elasticsearch-py
diff --git a/README.rst b/README.rst
new file mode 120000
index 0000000..100b938
--- /dev/null
+++ b/README.rst
@@ -0,0 +1 @@
+README
\ No newline at end of file
diff --git a/docs/api.rst b/docs/api.rst
index 1d4d086..f1ba4d9 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -63,6 +63,19 @@ connection class::
.. py:module:: elasticsearch
+Response Filtering
+~~~~~~~~~~~~~~~~~~
+
+The ``filter_path`` parameter is used to reduce the response returned by
+elasticsearch. For example, to only return ``_id`` and ``_type``, do::
+
+ es.search(index='test-index', filter_path=['hits.hits._id', 'hits.hits._type'])
+
+It also supports the ``*`` wildcard character to match any field or part of a
+field's name::
+
+ es.search(index='test-index', filter_path=['hits.hits._*'])
+
Elasticsearch
-------------
@@ -77,6 +90,12 @@ Indices
.. autoclass:: IndicesClient
:members:
+Ingest
+------
+
+.. autoclass:: IngestClient
+ :members:
+
Cluster
-------
@@ -96,8 +115,13 @@ Cat
:members:
Snapshot
----
+--------
.. autoclass:: SnapshotClient
:members:
+Tasks
+-----
+
+.. autoclass:: TasksClient
+ :members:
diff --git a/docs/conf.py b/docs/conf.py
index 7a946ca..38b0b26 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -49,10 +49,12 @@ copyright = u'2013, Honza Král'
# |version| and |release|, also used in various other places throughout the
# built documents.
#
+
+import elasticsearch
# The short X.Y version.
-version = '1.6.0'
+version = elasticsearch.__versionstr__
# The full version, including alpha/beta/rc tags.
-release = '1.6.0'
+release = version
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
@@ -228,7 +230,7 @@ latex_documents = [
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
- ('index', 'elasticsearch', u'Elasticsearch Documentation',
+ ('index', 'elasticsearch-py', u'Elasticsearch Documentation',
[u'Honza Král'], 1)
]
diff --git a/docs/connection.rst b/docs/connection.rst
index 26bc8b3..1c42f0c 100644
--- a/docs/connection.rst
+++ b/docs/connection.rst
@@ -3,7 +3,7 @@
Connection Layer API
====================
-All of the classes reponsible for handling the connection to the Elasticsearch
+All of the classes responsible for handling the connection to the Elasticsearch
cluster. The default subclasses used can be overriden by passing parameters to the
:class:`~elasticsearch.Elasticsearch` class. All of the arguments to the client
will be passed on to :class:`~elasticsearch.Transport`,
diff --git a/docs/exceptions.rst b/docs/exceptions.rst
index 906bb7a..1006722 100644
--- a/docs/exceptions.rst
+++ b/docs/exceptions.rst
@@ -21,4 +21,3 @@ Exceptions
.. autoclass:: NotFoundError(TransportError)
.. autoclass:: ConflictError(TransportError)
.. autoclass:: RequestError(TransportError)
-.. autoclass:: ConnectionError(TransportError)
diff --git a/docs/helpers.rst b/docs/helpers.rst
index 8fa220d..a477c6b 100644
--- a/docs/helpers.rst
+++ b/docs/helpers.rst
@@ -7,12 +7,93 @@ Collection of simple helper functions that abstract some specifics or the raw
API.
+Bulk helpers
+------------
+
+There are several helpers for the ``bulk`` API since it's requirement for
+specific formatting and other considerations can make it cumbersome if used directly.
+
+All bulk helpers accept an instance of ``Elasticsearch`` class and an iterable
+``actions`` (any iterable, can also be a generator, which is ideal in most
+cases since it will allow you to index large datasets without the need of
+loading them into memory).
+
+The items in the ``action`` iterable should be the documents we wish to index
+in several formats. The most common one is the same as returned by
+:meth:`~elasticsearch.Elasticsearch.search`, for example:
+
+.. code:: python
+
+ {
+ '_index': 'index-name',
+ '_type': 'document',
+ '_id': 42,
+ '_parent': 5,
+ 'pipeline': 'my-ingest-pipeline',
+ '_source': {
+ "title": "Hello World!",
+ "body": "..."
+ }
+ }
+
+Alternatively, if `_source` is not present, it will pop all metadata fields
+from the doc and use the rest as the document data:
+
+.. code:: python
+
+ {
+ "_id": 42,
+ "_parent": 5,
+ "title": "Hello World!",
+ "body": "..."
+ }
+
+The :meth:`~elasticsearch.Elasticsearch.bulk` api accepts ``index``, ``create``,
+``delete``, and ``update`` actions. Use the ``_op_type`` field to specify an
+action (``_op_type`` defaults to ``index``):
+
+.. code:: python
+
+ {
+ '_op_type': 'delete',
+ '_index': 'index-name',
+ '_type': 'document',
+ '_id': 42,
+ }
+ {
+ '_op_type': 'update',
+ '_index': 'index-name',
+ '_type': 'document',
+ '_id': 42,
+ 'doc': {'question': 'The life, universe and everything.'}
+ }
+
+
+.. note::
+
+ When reading raw json strings from a file, you can also pass them in
+ directly (without decoding to dicts first). In that case, however, you lose
+ the ability to specify anything (index, type, even id) on a per-record
+ basis, all documents will just be sent to elasticsearch to be indexed
+ as-is.
+
+
.. py:module:: elasticsearch.helpers
.. autofunction:: streaming_bulk
+.. autofunction:: parallel_bulk
+
.. autofunction:: bulk
+
+Scan
+----
+
.. autofunction:: scan
+
+Reindex
+-------
+
.. autofunction:: reindex
diff --git a/docs/index.rst b/docs/index.rst
index 7c7561b..aefd55d 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -9,13 +9,19 @@ For a more high level client library with more limited scope, have a look at
`elasticsearch-dsl`_ - it is a more pythonic library sitting on top of
``elasticsearch-py``.
-.. _elasticsearch-dsl: http://elasticsearch-dsl.rtfd.org/
+.. _elasticsearch-dsl: https://elasticsearch-dsl.readthedocs.io/
Compatibility
-------------
-The library is compatible with both Elasticsearch 1.x and 0.90.x but you
-**have to use a matching version**.
+The library is compatible with all Elasticsearch versions since ``0.90.x`` but you
+**have to use a matching major version**:
+
+For **Elasticsearch 5.0** and later, use the major version 5 (``5.x.y``) of the
+library.
+
+For **Elasticsearch 2.0** and later, use the major version 2 (``2.x.y``) of the
+library.
For **Elasticsearch 1.0** and later, use the major version 1 (``1.x.y``) of the
library.
@@ -26,13 +32,19 @@ library.
The recommended way to set your requirements in your `setup.py` or
`requirements.txt` is::
- # Elasticsearch 1.0
+ # Elasticsearch 5.x
+ elasticsearch>=5.0.0,<6.0.0
+
+ # Elasticsearch 2.x
+ elasticsearch>=2.0.0,<3.0.0
+
+ # Elasticsearch 1.x
elasticsearch>=1.0.0,<2.0.0
- # Elasticsearch 0.90
+ # Elasticsearch 0.90.x
elasticsearch<1.0.0
-The development is happening on ``master`` and ``0.4`` branches, respectively.
+The development is happening on ``master`` and ``2.x`` branches respectively.
Example Usage
-------------
@@ -46,7 +58,7 @@ Example Usage
doc = {
'author': 'kimchy',
'text': 'Elasticsearch: cool. bonsai cool.',
- 'timestamp': datetime(2010, 10, 10, 10, 10, 10)
+ 'timestamp': datetime.now(),
}
res = es.index(index="test-index", doc_type='tweet', id=1, body=doc)
print(res['created'])
@@ -65,18 +77,21 @@ Example Usage
Features
--------
-This client was designed as very thin wrapper around Elasticseach's REST API to
+This client was designed as very thin wrapper around Elasticsearch's REST API to
allow for maximum flexibility. This means that there are no opinions in this
client; it also means that some of the APIs are a little cumbersome to use from
-Python. We have created some :ref:`helpers` to help with this issue.
+Python. We have created some :ref:`helpers` to help with this issue as well as
+a more high level library (`elasticsearch-dsl`_) on top of this one to provide
+a more convenient way of working with Elasticsearch.
+
+.. _elasticsearch-dsl: https://elasticsearch-dsl.readthedocs.io/
Persistent Connections
~~~~~~~~~~~~~~~~~~~~~~
``elasticsearch-py`` uses persistent connections inside of individual connection
pools (one per each configured or sniffed node). Out of the box you can choose
-to use ``http``, ``thrift`` or an experimental ``memcached`` protocol to
-communicate with the elasticsearch nodes. See :ref:`transports` for more
+between two ``http`` protocol implementations. See :ref:`transports` for more
information.
The transport layer will create an instance of the selected connection class
@@ -103,7 +118,7 @@ If a connection to a node fails due to connection issues (raises
will be placed on hold for ``dead_timeout`` seconds and the request will be
retried on another node. If a connection fails multiple times in a row the
timeout will get progressively larger to avoid hitting a node that's, by all
-indication, down. If no live connection is availible, the connection that has
+indication, down. If no live connection is available, the connection that has
the smallest timeout will be used.
By default retries are not triggered by a timeout
@@ -131,8 +146,34 @@ Some example configurations::
es = Elasticsearch(["seed1", "seed2"], sniff_on_start=True)
# you can also sniff periodically and/or after failure:
- es = Elasticsearch(["seed1", "seed2"], sniff_on_start=True, sniff_on_connection_fail=True, sniffer_timeout=60)
+ es = Elasticsearch(["seed1", "seed2"],
+ sniff_on_start=True,
+ sniff_on_connection_fail=True,
+ sniffer_timeout=60)
+
+Thread safety
+~~~~~~~~~~~~~
+The client is thread safe and can be used in a multi threaded environment. Best
+practice is to create a single global instance of the client and use it
+throughout your application. If your application is long-running consider
+turning on :ref:`sniffing` to make sure the client is up to date on the cluster
+location.
+
+By default we allow ``urllib3`` to open up to 10 connections to each node, if
+your application calls for more parallelism, use the ``maxsize`` parameter to
+raise the limit::
+
+ # allow up to 25 connections to each node
+ es = Elasticsearch(["host1", "host2"], maxsize=25)
+
+.. note::
+
+ Since we use persistent connections throughout the client it means that the
+ client doesn't tolerate ``fork`` very well. If your application calls for
+ multiple processes make sure you create a fresh client after call to
+ ``fork``. Note that Python's ``multiprocessing`` module uses ``fork`` to
+ create new processes on POSIX systems.
SSL and Authentication
~~~~~~~~~~~~~~~~~~~~~~
@@ -154,18 +195,28 @@ elasticsearch cluster, including certificate verification and http auth::
['localhost', 'otherhost'],
http_auth=('user', 'secret'),
port=443,
- use_ssl=True,
- verify_certs=True,
- ca_certs=certifi.where(),
+ use_ssl=True
)
+ # SSL client authentication using client_cert and client_key
+
+ es = Elasticsearch(
+ ['localhost', 'otherhost'],
+ http_auth=('user', 'secret'),
+ port=443,
+ use_ssl=True,
+ ca_certs='/path/to/cacert.pem',
+ client_cert='/path/to/client_cert.pem',
+ client_key='/path/to/client_key.pem',
+ )
.. warning::
- By default SSL certificates won't be verified, pass in
- ``verify_certs=True`` to make sure your certificates will get verified. The
- client doesn't ship with any CA certificates; easiest way to obtain the
- common set is by using the `certifi`_ package (as shown above).
+ ``elasticsearch-py`` doesn't ship with default set of root certificates. To
+ have working SSL certificate validation you need to either specify your own
+ as ``ca_certs`` or install `certifi`_ which will be picked up
+ automatically.
+
See class :class:`~elasticsearch.Urllib3HttpConnection` for detailed
description of the options.
@@ -180,8 +231,10 @@ two loggers: ``elasticsearch`` and ``elasticsearch.trace``. ``elasticsearch``
is used by the client to log standard activity, depending on the log level.
``elasticsearch.trace`` can be used to log requests to the server in the form
of ``curl`` commands using pretty-printed json that can then be executed from
-command line. If the trace logger has not been configured already it is set to
-`propagate=False` so it needs to be activated separately.
+command line. Because it is designed to be shared (for example to demonstrate
+an issue) it also just uses ``localhost:9200`` as the address instead of the
+actual address of the host. If the trace logger has not been configured
+already it is set to `propagate=False` so it needs to be activated separately.
.. _logging library: http://docs.python.org/3.3/library/logging.html
@@ -193,7 +246,7 @@ could come into play.
When using an http load balancer you cannot use the :ref:`sniffing`
functionality - the cluster would supply the client with IP addresses to
-directly cnnect to the cluster, circumventing the load balancer. Depending on
+directly connect to the cluster, circumventing the load balancer. Depending on
your configuration this might be something you don't want or break completely.
In some environments (notably on Google App Engine) your http requests might be
@@ -204,6 +257,29 @@ bodies via post::
from elasticsearch import Elasticsearch
es = Elasticsearch(send_get_body_as='POST')
+Running on AWS with IAM
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you want to use this client with IAM based authentication on AWS you can use
+the `requests-aws4auth`_ package::
+
+ from elasticsearch import Elasticsearch, RequestsHttpConnection
+ from requests_aws4auth import AWS4Auth
+
+ host = 'YOURHOST.us-east-1.es.amazonaws.com'
+ awsauth = AWS4Auth(YOUR_ACCESS_KEY, YOUR_SECRET_KEY, REGION, 'es')
+
+ es = Elasticsearch(
... 7776 lines suppressed ...
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/python-elasticsearch.git
More information about the Python-modules-commits
mailing list