[Python-modules-commits] [elasticsearch-curator] 03/07: Import elasticsearch-curator_3.3.0.orig.tar.gz
Apollon Oikonomopoulos
apoikos at moszumanska.debian.org
Fri Oct 23 13:10:05 UTC 2015
This is an automated email from the git hooks/post-receive script.
apoikos pushed a commit to tag debian/3.3.0-1
in repository elasticsearch-curator.
commit e703ed91c99b143acba406c96a67aefd4ef81869
Author: Apollon Oikonomopoulos <apoikos at debian.org>
Date: Fri Oct 23 15:56:32 2015 +0300
Import elasticsearch-curator_3.3.0.orig.tar.gz
---
CONTRIBUTORS | 1 +
Changelog.rst | 41 ++++++++++++++++
curator/_version.py | 2 +-
curator/api/allocation.py | 19 +++++---
curator/api/filter.py | 21 +++++---
curator/api/utils.py | 7 +--
curator/cli/allocation.py | 9 +++-
curator/cli/cli.py | 12 ++++-
curator/cli/snapshot_selection.py | 6 +--
curator/cli/utils.py | 24 ++++++---
docs/asciidoc/commands/allocation.asciidoc | 16 +++---
docs/asciidoc/commands/optimize.asciidoc | 2 +-
docs/asciidoc/flags/certificate.asciidoc | 28 +++++++++++
docs/asciidoc/flags/index.asciidoc | 4 ++
docs/asciidoc/flags/newer-than.asciidoc | 5 +-
docs/asciidoc/flags/older-than.asciidoc | 3 ++
docs/asciidoc/flags/ssl-no-validate.asciidoc | 34 +++++++++++++
docs/asciidoc/flags/use_ssl.asciidoc | 38 +++++++++++++--
docs/asciidoc/index.asciidoc | 2 +-
setup.py | 2 +-
test/integration/test_api_commands.py | 15 ++++++
test/integration/test_cli_commands.py | 73 +++++++++++++++++++++++++++-
test/integration/test_cli_utils.py | 42 +++++++++++++++-
test/unit/test_api_commands.py | 14 ++++++
test/unit/test_api_filter.py | 22 +++++++++
test/unit/test_cli_utils.py | 9 ++++
26 files changed, 405 insertions(+), 46 deletions(-)
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 83a544c..360aefa 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -61,3 +61,4 @@ Contributors:
* Mark Feltner (feltnerm)
* William Jimenez (wjimenez5271)
* Jeremy Canady (jrmycanady)
+* Steven Ottenhoff (steffo)
diff --git a/Changelog.rst b/Changelog.rst
index b440da0..5f1e02c 100644
--- a/Changelog.rst
+++ b/Changelog.rst
@@ -3,6 +3,47 @@
Changelog
=========
+3.3.0 (31 August 2015)
+----------------------
+
+**Announcement**
+
+ * Curator is tested in Jenkins. Each commit to the master branch is tested
+ with both Python versions 2.7.6 and 3.4.0 against each of the following
+ Elasticsearch versions:
+ * 1.7_nightly
+ * 1.6_nightly
+ * 1.7.0
+ * 1.6.1
+ * 1.5.1
+ * 1.4.4
+ * 1.3.9
+ * 1.2.4
+ * 1.1.2
+ * 1.0.3
+ * If you are using a version different from this, your results may vary.
+
+**General**
+
+ * Allocation type can now also be ``include`` or ``exclude``, in addition to the
+ the existing default ``require`` type. Add ``--type`` to the allocation command
+ to specify the type. #443 (steffo)
+
+ * Bump elasticsearch python module dependency to 1.6.0+ to enable synced_flush
+ API call. Reported in #447 (untergeek)
+
+ * Add SSL features, ``--ssl-no-validate`` and ``certificate`` to provide other
+ ways to validate SSL connections to Elasticsearch. #436 (untergeek)
+
+**Bug fixes**
+
+ * Delete by space was only reporting space used by primary shards. Fixed to
+ show all space consumed. Reported in #455 (untergeek)
+
+ * Update exit codes and messages for snapshot selection. Reported in #452 (untergeek)
+
+ * Fix potential int/float casting issues. Reported in #465 (untergeek)
+
3.2.3 (16 July 2015)
--------------------
diff --git a/curator/_version.py b/curator/_version.py
index 3220610..6a157dc 100644
--- a/curator/_version.py
+++ b/curator/_version.py
@@ -1 +1 @@
-__version__ = '3.2.3'
+__version__ = '3.3.0'
diff --git a/curator/api/allocation.py b/curator/api/allocation.py
index 24a180f..33ba94c 100644
--- a/curator/api/allocation.py
+++ b/curator/api/allocation.py
@@ -3,7 +3,7 @@ import elasticsearch
import logging
logger = logging.getLogger(__name__)
-def apply_allocation_rule(client, indices, rule=None):
+def apply_allocation_rule(client, indices, rule=None, allocation_type='require' ):
"""
Apply a required allocation rule to a list of indices.
@@ -12,6 +12,7 @@ def apply_allocation_rule(client, indices, rule=None):
:arg rule: The routing allocation rule to apply, e.g. ``tag=ssd``. Must be
in the format of ``key=value``, and should match values declared on the
correlating nodes in your cluster.
+ :arg allocation_type: Type of allocation to apply
:rtype: bool
.. note::
@@ -23,21 +24,26 @@ def apply_allocation_rule(client, indices, rule=None):
return False
key = rule.split('=')[0]
value = rule.split('=')[1]
- indices = prune_allocated(client, indices, key, value)
+ indices = prune_allocated(client, indices, key, value, allocation_type)
+
+ if allocation_type not in ['require', 'include', 'exclude']:
+ logger.error("{0} is an invalid allocation_type. Must be one of 'require', 'include', 'exclude'.".format(allocation_type))
+ return False
+
if not indices:
logger.warn("No indices to act on.")
return False
- logger.info('Updating index setting index.routing.allocation.require.{0}={1}'.format(key,value))
+ logger.info('Updating index setting index.routing.allocation.{0}.{1}={2}'.format(allocation_type,key,value))
try:
client.indices.put_settings(index=to_csv(indices),
- body='index.routing.allocation.require.{0}={1}'.format(key,value),
+ body='index.routing.allocation.{0}.{1}={2}'.format(allocation_type,key,value),
)
return True
except:
logger.error("Error in updating index settings with allocation rule. Run with --debug flag and/or check Elasticsearch logs for more information.")
return False
-def allocation(client, indices, rule=None):
+def allocation(client, indices, rule=None, allocation_type='require' ):
"""
Helper method called by the CLI.
@@ -46,6 +52,7 @@ def allocation(client, indices, rule=None):
:arg rule: The routing allocation rule to apply, e.g. ``tag=ssd``. Must be
in the format of ``key=value``, and should match values declared on the
correlating nodes in your cluster.
+ :arg allocation_type: Type of allocation to apply
:rtype: bool
"""
- return apply_allocation_rule(client, indices, rule=rule)
+ return apply_allocation_rule(client, indices, rule=rule, allocation_type=allocation_type )
diff --git a/curator/api/filter.py b/curator/api/filter.py
index 320a2f3..19bf130 100644
--- a/curator/api/filter.py
+++ b/curator/api/filter.py
@@ -67,7 +67,7 @@ def build_filter(
if not timestring:
logger.error("older_than and newer_than require timestring parameter")
return {}
- argdict = { "groupname":groupname, "time_unit":time_unit,
+ argdict = { "groupname":groupname, "time_unit":time_unit,
"timestring": timestring, "value": value,
"method": kindOf }
date_regex = get_date_regex(timestring)
@@ -280,6 +280,10 @@ def timestamp_check(timestamp, timestring=None, time_unit=None,
"""
cutoff = get_cutoff(unit_count=value, time_unit=time_unit, utc_now=utc_now)
+ if not cutoff:
+ logger.error('No cutoff value.')
+ return False
+
try:
object_time = get_datetime(timestamp, timestring)
except ValueError:
@@ -319,13 +323,18 @@ def filter_by_space(client, indices, disk_space=None, reverse=True):
"""
def get_stat_list(stats):
- retval = list(
- (index_name, index_stats['index']['primary_size_in_bytes'])
- for (index_name, index_stats) in stats['indices'].items()
- )
+ retval = []
+ for (index_name, index_stats) in stats['indices'].items():
+ replica_count = client.indices.get_settings(index_name)[index_name]['settings']['index']['number_of_replicas']
+ size = "size_in_bytes" if int(replica_count) > 0 else "primary_size_in_bytes"
+ logger.debug('Index: {0} Size: {1}'.format(index_name, index_stats['index'][size]))
+ retval.append((index_name, index_stats['index'][size]))
return retval
- if not disk_space:
+ # Ensure that disk_space is a float
+ if disk_space:
+ disk_space = float(disk_space)
+ else:
logger.error("Mising value for disk_space.")
return False
diff --git a/curator/api/utils.py b/curator/api/utils.py
index ebb7fcc..9fafece 100644
--- a/curator/api/utils.py
+++ b/curator/api/utils.py
@@ -301,7 +301,7 @@ def prune_closed(client, indices):
logger.info('Skipping index {0}: Closed.'.format(idx))
return sorted(retval)
-def prune_allocated(client, indices, key, value):
+def prune_allocated(client, indices, key, value, allocation_type):
"""
Return list of indices that do not have the routing allocation rule of
`key=value`
@@ -310,6 +310,7 @@ def prune_allocated(client, indices, key, value):
:arg indices: A list of indices to act on
:arg key: The allocation attribute to check for
:arg value: The value to check for
+ :arg allocation_type: Type of allocation to apply
:rtype: list
"""
indices = ensure_list(indices)
@@ -319,11 +320,11 @@ def prune_allocated(client, indices, key, value):
index=idx,
)
try:
- has_routing = settings[idx]['settings']['index']['routing']['allocation']['require'][key] == value
+ has_routing = settings[idx]['settings']['index']['routing']['allocation'][allocation_type][key] == value
except KeyError:
has_routing = False
if has_routing:
- logger.debug('Skipping index {0}: Already has allocation rule {1} applied.'.format(idx, key + "=" + value))
+ logger.debug('Skipping index {0}: Allocation rule {1} is already applied for type {2}.'.format(idx, key + "=" + value, allocation_type))
else:
retval.append(idx)
return sorted(retval)
diff --git a/curator/cli/allocation.py b/curator/cli/allocation.py
index 7b67938..c9b9c9f 100644
--- a/curator/cli/allocation.py
+++ b/curator/cli/allocation.py
@@ -8,11 +8,18 @@ logger = logging.getLogger(__name__)
@cli.group('allocation')
@click.option('--rule', show_default=True, expose_value=True, type=str,
help='Routing allocation rule to apply, e.g. tag=ssd')
+ at click.option('--type', default='require', show_default=True, expose_value=True, type=str,
+ help='Specify an allocation type, include, exclude or require')
@click.pass_context
-def allocation(ctx, rule):
+def allocation(ctx, rule, type):
"""Index Allocation"""
if not rule:
click.echo('{0}'.format(ctx.get_help()))
click.echo(click.style('Missing required parameter --rule', fg='red', bold=True))
sys.exit(1)
+
+ if type not in ['require', 'include', 'exclude']:
+ click.echo(click.style('--type can only be one of: require, include exclude', fg='red', bold=True))
+ sys.exit(1)
+
allocation.add_command(indices)
diff --git a/curator/cli/cli.py b/curator/cli/cli.py
index 6239cb0..65ae2ff 100644
--- a/curator/cli/cli.py
+++ b/curator/cli/cli.py
@@ -34,6 +34,8 @@ DEFAULT_ARGS = {
@click.option('--url_prefix', help='Elasticsearch http url prefix.', default=DEFAULT_ARGS['url_prefix'])
@click.option('--port', help='Elasticsearch port.', default=DEFAULT_ARGS['port'], type=int)
@click.option('--use_ssl', help='Connect to Elasticsearch through SSL.', is_flag=True, default=DEFAULT_ARGS['use_ssl'])
+ at click.option('--certificate', help='Path to certificate to use for SSL validation. (OPTIONAL)', type=str, default=None)
+ at click.option('--ssl-no-validate', help='Do not validate SSL certificate', is_flag=True)
@click.option('--http_auth', help='Use Basic Authentication ex: user:pass', default=DEFAULT_ARGS['http_auth'])
@click.option('--timeout', help='Connection timeout in seconds.', default=DEFAULT_ARGS['timeout'], type=int)
@click.option('--master-only', is_flag=True, help='Only operate on elected master node.')
@@ -44,7 +46,7 @@ DEFAULT_ARGS = {
@click.option('--logformat', help='Log output format [default|logstash].', default=DEFAULT_ARGS['logformat'])
@click.version_option(version=__version__)
@click.pass_context
-def cli(ctx, host, url_prefix, port, use_ssl, http_auth, timeout, master_only, dry_run, debug, loglevel, logfile, logformat):
+def cli(ctx, host, url_prefix, port, use_ssl, certificate, ssl_no_validate, http_auth, timeout, master_only, dry_run, debug, loglevel, logfile, logformat):
"""
Curator for Elasticsearch indices.
@@ -70,6 +72,14 @@ def cli(ctx, host, url_prefix, port, use_ssl, http_auth, timeout, master_only, d
logging.root.addHandler(handler)
logging.root.setLevel(numeric_log_level)
+ # Test whether certificate is a valid file path
+ if use_ssl is True and certificate is not None:
+ try:
+ open(certificate, 'r')
+ except IOError:
+ click.echo(click.style('Error: Could not open certificate at {0}'.format(certificate), fg='red', bold=True))
+ sys.exit(1)
+
# Filter out logging from Elasticsearch and associated modules by default
if not debug:
for handler in logging.root.handlers:
diff --git a/curator/cli/snapshot_selection.py b/curator/cli/snapshot_selection.py
index 2d701f3..4522b07 100644
--- a/curator/cli/snapshot_selection.py
+++ b/curator/cli/snapshot_selection.py
@@ -59,8 +59,8 @@ def snapshots(ctx, newer_than, older_than, prefix, suffix, time_unit,
if snapshots:
working_list = snapshots
else:
- click.echo(click.style('ERROR. No snapshots found in Elasticsearch.', fg='red', bold=True))
- sys.exit(1)
+ click.echo(click.style('No snapshots found in Elasticsearch.', fg='red', bold=True))
+ sys.exit(0)
if all_snapshots:
logger.info('Matching all snapshots. Ignoring flags other than --exclude.')
@@ -96,5 +96,5 @@ def snapshots(ctx, newer_than, older_than, prefix, suffix, time_unit,
else:
logger.warn('No snapshots matched provided args.')
- click.echo(click.style('ERROR. No snapshots matched provided args.', fg='red', bold=True))
+ click.echo(click.style('No snapshots matched provided args.', fg='red', bold=True))
sys.exit(0)
diff --git a/curator/cli/utils.py b/curator/cli/utils.py
index 5d615f1..5e7b785 100644
--- a/curator/cli/utils.py
+++ b/curator/cli/utils.py
@@ -107,16 +107,26 @@ def get_client(**kwargs):
"""
kwargs['master_only'] = False if not 'master_only' in kwargs else kwargs['master_only']
kwargs['use_ssl'] = False if not 'use_ssl' in kwargs else kwargs['use_ssl']
+ kwargs['ssl_no_validate'] = False if not 'ssl_no_validate' in kwargs else kwargs['ssl_no_validate']
+ kwargs['certificate'] = False if not 'certificate' in kwargs else kwargs['certificate']
logger.debug("kwargs = {0}".format(kwargs))
master_only = kwargs.pop('master_only')
if kwargs['use_ssl']:
- try:
+ if kwargs['ssl_no_validate']:
+ kwargs['verify_certs'] = False # Not needed, but explicitly defined
+ else:
logger.info('Attempting to verify SSL certificate.')
- import certifi
- kwargs['verify_certs'] = True
- kwargs['ca_certs'] = certifi.where()
- except ImportError:
- logger.warn('Unable to verify SSL certificate.')
+ # If user provides a certificate:
+ if kwargs['certificate']:
+ kwargs['verify_certs'] = True
+ kwargs['ca_certs'] = kwargs['certificate']
+ else: # Try to use certifi certificates:
+ try:
+ import certifi
+ kwargs['verify_certs'] = True
+ kwargs['ca_certs'] = certifi.where()
+ except ImportError:
+ logger.warn('Unable to verify SSL certificate.')
try:
client = elasticsearch.Elasticsearch(**kwargs)
# Verify the version is acceptable.
@@ -210,7 +220,7 @@ def do_command(client, command, indices, params=None, master_timeout=30000):
client, indices, alias=params['name'], remove=params['remove']
)
if command == "allocation":
- return allocation(client, indices, rule=params['rule'])
+ return allocation(client, indices, rule=params['rule'], allocation_type=params['type'] )
if command == "bloom":
return bloom(client, indices, delay=params['delay'])
if command == "close":
diff --git a/docs/asciidoc/commands/allocation.asciidoc b/docs/asciidoc/commands/allocation.asciidoc
index 43680c7..91d5b1a 100644
--- a/docs/asciidoc/commands/allocation.asciidoc
+++ b/docs/asciidoc/commands/allocation.asciidoc
@@ -45,8 +45,9 @@ ________________________________________________________________________________
-It is this last method, `index.routing.allocation.require.*`, that Curator uses
-to force routing. Curator does not use `include` or `exclude` tagging.
+In versions 3.2.3 and older, Curator used this last method,
+`index.routing.allocation.require.*`, to force routing. Beginning in version
+3.3.0, Curator also allows the use of `include` and `exclude` tagging.
[float]
Use-case
@@ -98,14 +99,17 @@ Flags
$ curator allocation --help
Usage: curator allocation [OPTIONS] COMMAND [ARGS]...
-Index Allocation
+ Index Allocation
Options:
- --rule TEXT Routing allocation rule to apply, e.g. tag=ssd
- --help Show this message and exit.
+ --rule TEXT Routing allocation rule to apply, e.g. tag=ssd
+ --type TEXT Specify an allocation type, include, exclude or require
+ [default: require]
+ --help Show this message and exit.
Commands:
- indices Index selection.
+ indices Index selection.
+
--------------------------------------------------------------------------------
diff --git a/docs/asciidoc/commands/optimize.asciidoc b/docs/asciidoc/commands/optimize.asciidoc
index 352a8aa..3ea5565 100644
--- a/docs/asciidoc/commands/optimize.asciidoc
+++ b/docs/asciidoc/commands/optimize.asciidoc
@@ -58,7 +58,7 @@ Options:
--max_num_segments INTEGER Merge to this number of segments per shard.
[default: 2]
--request_timeout INTEGER Allow this many seconds before the transaction
- times out. [default: 218600]
+ times out. [default: 21600]
--help Show this message and exit.
Commands:
diff --git a/docs/asciidoc/flags/certificate.asciidoc b/docs/asciidoc/flags/certificate.asciidoc
new file mode 100644
index 0000000..3cc9862
--- /dev/null
+++ b/docs/asciidoc/flags/certificate.asciidoc
@@ -0,0 +1,28 @@
+[[certificate]]
+== --certificate
+
+[float]
+Summary
+~~~~~~~
+
+Allows the use of a specified CA certificate file to validate the SSL certificate
+used by Elasticsearch.
+
+[float]
+Flags
+~~~~~
+
+* `--certificate` Path to certificate to use for SSL validation.
+
+IMPORTANT: This flag must come before any <<commands,command>>.
+
+[float]
+Example
+~~~~~~~
+
+Connect to a cluster at `https://example.com/` via SSL and verify the
+certificate with the provided CA certificate file:
+
+---------------------------------------------------------------------
+curator --host example.com --port 443 --use_ssl --certificate /path/to/cacert.pem <<command>> <<flags>>
+---------------------------------------------------------------------
diff --git a/docs/asciidoc/flags/index.asciidoc b/docs/asciidoc/flags/index.asciidoc
index 839294a..3ec7e7a 100644
--- a/docs/asciidoc/flags/index.asciidoc
+++ b/docs/asciidoc/flags/index.asciidoc
@@ -17,6 +17,8 @@
* <<timeout,--timeout>>
* <<url_prefix,--url_prefix>>
* <<use_ssl,--use_ssl>>
+* <<certificate,--certificate>>
+* <<ssl-no-validate,--ssl-no-validate>>
* <<version,--version>>
_Index and Snapshot Selection_
@@ -59,6 +61,8 @@ include::port.asciidoc[]
include::timeout.asciidoc[]
include::url_prefix.asciidoc[]
include::use_ssl.asciidoc[]
+include::certificate.asciidoc[]
+include::ssl-no-validate.asciidoc[]
include::version.asciidoc[]
include::newer-than.asciidoc[]
diff --git a/docs/asciidoc/flags/newer-than.asciidoc b/docs/asciidoc/flags/newer-than.asciidoc
index a46bc72..b8142fa 100644
--- a/docs/asciidoc/flags/newer-than.asciidoc
+++ b/docs/asciidoc/flags/newer-than.asciidoc
@@ -8,9 +8,12 @@ Summary
IMPORTANT: This flag only functions within the scope of <<index-selection,index selection>> or <<snapshot-selection,snapshot selection>>.
-Filter indices "newer than" a given number of <<time-unit,time-units>>. The
+Filter indices "newer than" the given number of <<time-unit,time-units>>. The
indices must also have a <<timestring>>.
+The value provided indicates a given number of <<time-unit,time-units>> ago to
+use as a reference point. All indices "newer than" that point will be included.
+
[float]
Flags
~~~~~
diff --git a/docs/asciidoc/flags/older-than.asciidoc b/docs/asciidoc/flags/older-than.asciidoc
index 8ec388c..5917fb7 100644
--- a/docs/asciidoc/flags/older-than.asciidoc
+++ b/docs/asciidoc/flags/older-than.asciidoc
@@ -11,6 +11,9 @@ IMPORTANT: This flag only functions within the scope of <<index-selection,index
Filter indices "older than" a given number of <<time-unit,time-units>>. The
indices must also have a <<timestring>>.
+The value provided indicates a given number of <<time-unit,time-units>> ago to
+use as a reference point. All indices "older than" that point will be included.
+
[float]
Flags
~~~~~
diff --git a/docs/asciidoc/flags/ssl-no-validate.asciidoc b/docs/asciidoc/flags/ssl-no-validate.asciidoc
new file mode 100644
index 0000000..223e4f1
--- /dev/null
+++ b/docs/asciidoc/flags/ssl-no-validate.asciidoc
@@ -0,0 +1,34 @@
+[[ssl-no-validate]]
+== --ssl-no-validate
+
+[float]
+Summary
+~~~~~~~
+
+If access to your Elasticsearch instance is protected by SSL encryption, you
+may use the `--ssl-no-validate` flag to disable SSL certificate verification.
+
+Valid use cases for this flag include the use of self-signed certificates that
+cannot be otherwise verified and would generate error messages.
+
+NOTE: The use of this flag will likely result in a warning message that your SSL
+certificates are not trusted. This is expected behavior.
+
+[float]
+Flags
+~~~~~
+
+* `--ssl-no-validate` Do not validate SSL certificate
+
+IMPORTANT: This flag must come before any <<commands,command>>.
+
+[float]
+Example
+~~~~~~~
+
+Connect to a cluster at `https://example.com/` via SSL but do not verify the
+certificate:
+
+---------------------------------------------------------------------
+curator --host example.com --port 443 --use_ssl --ssl-no-validate <<command>> <<flags>>
+---------------------------------------------------------------------
diff --git a/docs/asciidoc/flags/use_ssl.asciidoc b/docs/asciidoc/flags/use_ssl.asciidoc
index 5c5bc4d..a84b77c 100644
--- a/docs/asciidoc/flags/use_ssl.asciidoc
+++ b/docs/asciidoc/flags/use_ssl.asciidoc
@@ -8,11 +8,39 @@ Summary
If access to your Elasticsearch instance is protected by SSL encryption, you
must use the `--use_ssl` flag.
-TIP: Beginning with version 3.2.0, Curator has _experimental_ support for
-validating SSL certificates. In order to make this work, an additional Python
-module needs to be installed via `pip install certifi`. If the `certifi` module
-is installed, and you are connecting to Elasticsearch via SSL, the verification
-should automatically take place.
+[float]
+SSL Certificate Validation
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Beginning with version 3.2.0, Curator added _experimental_ support for
+validating SSL certificates. These features were enhanced in version 3.3.0 to
+support additional SSL validation options.
+
+Depending on which validation option you choose, you may need to install an
+additional Python module.
+
+**If you have certificates signed by a commonly known Certificate Authority...**
+
+Chances are good that the CA certificates bundled with Mozilla Firefox will be
+able to validate your certificate. In this case, use of the `certifi` python
+module will work. This can be installed via `pip install certifi`.
+
+With the `certifi` module installed, if you are connecting to Elasticsearch via
+SSL, the verification should automatically take place.
+
+**If you have an unrecognized CA, use self-signed certificates, or do not wish
+to install the `certifi` module...**
+
+In this case you can use the <<certificate,--certificate>> flag to point to the
+file which holds the CA certificate to use for validation. This file should be in
+PEM format.
+
+**If you do not wish to validate your certificate at all...**
+
+This is not recommended, but if you so desire, you can use the
+<<ssl-no-validate,--ssl-no-validate>> flag. Use of this flag will likely still
+yield a warning about insecure connections, but still permit the use of SSL
+communications.
[float]
Flags
diff --git a/docs/asciidoc/index.asciidoc b/docs/asciidoc/index.asciidoc
index 30d4bc8..938a5bd 100644
--- a/docs/asciidoc/index.asciidoc
+++ b/docs/asciidoc/index.asciidoc
@@ -1,4 +1,4 @@
-:curator_version: 3.2.2
+:curator_version: 3.2.3
:curator_major: 3
:es_py_version: 1.6.0
:ref: http://www.elastic.co/guide/en/elasticsearch/reference/current
diff --git a/setup.py b/setup.py
index fd43444..a089784 100644
--- a/setup.py
+++ b/setup.py
@@ -22,7 +22,7 @@ def get_version():
return VERSION
def get_install_requires():
- res = ['elasticsearch>=1.0.0,<2.0.0' ]
+ res = ['elasticsearch>=1.6.0,<2.0.0' ]
res.append('click>=3.3')
return res
diff --git a/test/integration/test_api_commands.py b/test/integration/test_api_commands.py
index 3bff435..ab40218 100644
--- a/test/integration/test_api_commands.py
+++ b/test/integration/test_api_commands.py
@@ -123,6 +123,20 @@ class TestChangeAllocation(CuratorTestCase):
self.assertEquals('value', self.client.indices.get_settings(index='test_index')['test_index']['settings']['index']['routing']['allocation']['require']['key'])
+ def test_index_allocation_can_be_modified_for_include(self):
+ self.create_index('test_index')
+
+ curator.apply_allocation_rule(self.client, 'test_index', rule="key=value", allocation_type='include')
+
+ self.assertEquals('value', self.client.indices.get_settings(index='test_index')['test_index']['settings']['index']['routing']['allocation']['include']['key'])
+
+ def test_index_allocation_can_be_modified_for_exclude(self):
+ self.create_index('test_index')
+
+ curator.apply_allocation_rule(self.client, 'test_index', rule="key=value", allocation_type='exclude')
+
+ self.assertEquals('value', self.client.indices.get_settings(index='test_index')['test_index']['settings']['index']['routing']['allocation']['exclude']['key'])
+
def test_closed_index_allocation_can_be_modified(self):
self.create_index('test_index')
@@ -146,6 +160,7 @@ class TestChangeAllocation(CuratorTestCase):
self.assertEquals('value', self.client.indices.get_settings(index='test_index')['test_index']['settings']['index']['routing']['allocation']['require']['key'])
+
class TestDeleteIndex(CuratorTestCase):
def test_index_will_be_deleted(self):
self.create_index('test_index')
diff --git a/test/integration/test_cli_commands.py b/test/integration/test_cli_commands.py
index e9caeed..379f96e 100644
--- a/test/integration/test_cli_commands.py
+++ b/test/integration/test_cli_commands.py
@@ -263,6 +263,75 @@ class TestCLIAllocation(CuratorTestCase):
self.client.indices.get_settings(index='my_index')['my_index']['settings']['index']['routing']['allocation']['require'][key]
)
+ def test_allocation_all_indices_include(self):
+ self.create_index('my_index')
+ key = 'foo'
+ value = 'bar'
+ rule = key + '=' + value
+ allocation_type = 'include'
+ test = clicktest.CliRunner()
+ result = test.invoke(
+ curator.cli,
+ [
+ '--logfile', os.devnull,
+ '--host', host,
+ '--port', str(port),
+ 'allocation', '--rule', rule,
+ '--type', allocation_type,
+ 'indices',
+ '--all-indices',
+ ],
+ obj={"filters":[]})
+ self.assertEquals(
+ value,
+ self.client.indices.get_settings(index='my_index')['my_index']['settings']['index']['routing']['allocation'][allocation_type][key]
+ )
+
+ def test_allocation_all_indices_exclude(self):
+ self.create_index('my_index')
+ key = 'foo'
+ value = 'bar'
+ rule = key + '=' + value
+ allocation_type = 'exclude'
+ test = clicktest.CliRunner()
+ result = test.invoke(
+ curator.cli,
+ [
+ '--logfile', os.devnull,
+ '--host', host,
+ '--port', str(port),
+ 'allocation', '--rule', rule,
+ '--type', allocation_type,
+ 'indices',
+ '--all-indices',
+ ],
+ obj={"filters":[]})
+ self.assertEquals(
+ value,
+ self.client.indices.get_settings(index='my_index')['my_index']['settings']['index']['routing']['allocation'][allocation_type][key]
+ )
+
+ def test_allocation_fail_on_bad_type(self):
+ self.create_index('my_index')
+ key = 'foo'
+ value = 'bar'
+ rule = key + '=' + value
+ allocation_type = 'fail'
+ test = clicktest.CliRunner()
+ result = test.invoke(
+ curator.cli,
+ [
+ '--logfile', os.devnull,
+ '--host', host,
+ '--port', str(port),
+ 'allocation', '--rule', rule,
+ '--type', allocation_type,
+ 'indices',
+ '--all-indices',
+ ],
+ obj={"filters":[]})
+ self.assertEqual(1, result.exit_code)
+
class TestCLIBloom(CuratorTestCase):
def test_bloom_cli(self):
self.create_indices(5)
@@ -457,7 +526,7 @@ class TestCLIOptimize(CuratorTestCase):
body={"doc" + i :'TEST DOCUMENT'},
)
# This should force each doc to be in its own segment.
- self.client.indices.flush(index="index_name", force=True, full=True)
+ self.client.indices.flush(index="index_name", force=True)
test = clicktest.CliRunner()
result = test.invoke(
@@ -716,7 +785,7 @@ class TestCLISnapshotSelection(CuratorTestCase):
'--all-snapshots',
],
obj={"filters":[]})
- self.assertEqual(1, result.exit_code)
+ self.assertEqual(0, result.exit_code)
def test_snapshot_selection_show_filtered(self):
self.create_repository()
for i in ["1", "2", "3"]:
diff --git a/test/integration/test_cli_utils.py b/test/integration/test_cli_utils.py
index f7c5d73..31f96b5 100644
--- a/test/integration/test_cli_utils.py
+++ b/test/integration/test_cli_utils.py
@@ -55,4 +55,44 @@ class TestCLIUtilsFilterCallback(CuratorTestCase):
],
obj={"filters":[]})
self.assertEqual(1, result.exit_code)
-
+
+class TestSSLFlags(CuratorTestCase):
+ def test_bad_certificate(self):
+ self.create_indices(10)
+ test = clicktest.CliRunner()
+ result = test.invoke(
+ curator.cli,
+ [
+ '--logfile', os.devnull,
+ '--host', host,
+ '--port', str(port),
+ '--use_ssl',
+ '--certificate', '/path/to/nowhere',
+ 'show',
+ 'indices',
+ '--older-than', '5',
+ '--time-unit', 'days',
+ '--timestring', '%Y.%m.%d',
+ ],
+ obj={"filters":[]})
+ self.assertEqual(1, result.exit_code)
+ self.assertEqual('Error: Could not open certificate at /path/to/nowhere\n', result.output)
+ def test_ssl_no_validate(self):
+ self.create_indices(10)
+ test = clicktest.CliRunner()
+ result = test.invoke(
+ curator.cli,
+ [
+ '--logfile', os.devnull,
+ '--host', host,
+ '--port', str(port),
+ '--use_ssl',
+ '--ssl-no-validate',
+ 'show',
+ 'indices',
+ '--older-than', '5',
+ '--time-unit', 'days',
+ '--timestring', '%Y.%m.%d',
+ ],
+ obj={"filters":[]})
+ self.assertTrue(1, result.exit_code)
diff --git a/test/unit/test_api_commands.py b/test/unit/test_api_commands.py
index 1fe1356..abcc5da 100644
--- a/test/unit/test_api_commands.py
+++ b/test/unit/test_api_commands.py
@@ -197,6 +197,20 @@ class TestAllocate(TestCase):
client.indices.get_settings.return_value = allocation_out
client.indices.put_settings.side_effect = fake_fail
self.assertFalse(curator.allocation(client, named_index, rule="foo=bar"))
+ def test_allocation_wrong_type_param(self):
+ client = Mock()
+ client.cluster.state.return_value = open_index
+ client.indices.get_settings.return_value = allocation_out
+ client.indices.put_settings.return_value = None
+ self.assertFalse(curator.allocation(client, named_index, rule="foo=bar", allocation_type="wrong_type"))
+ def test_allocation_good_type_param(self):
+ client = Mock()
+ client.cluster.state.return_value = open_index
+ client.indices.get_settings.return_value = allocation_out
+ client.indices.put_settings.return_value = None
+ self.assertTrue(curator.allocation(client, named_index, rule="foo=bar", allocation_type="require"))
+ self.assertTrue(curator.allocation(client, named_index, rule="foo=bar", allocation_type="include"))
+ self.assertTrue(curator.allocation(client, named_index, rule="foo=bar", allocation_type="exclude"))
class TestBloom(TestCase):
def test_disable_bloom_no_more_bloom_positive(self):
diff --git a/test/unit/test_api_filter.py b/test/unit/test_api_filter.py
index 4137f6b..1fda5aa 100644
--- a/test/unit/test_api_filter.py
+++ b/test/unit/test_api_filter.py
@@ -30,6 +30,9 @@ re_test_indices = [
"logstashfoo", "logstashbar",
]
+sized = { u'index1': {u'settings': {u'index': {u'number_of_replicas': u'0'} } },
+ u'index2': {u'settings': {u'index': {u'number_of_replicas': u'0'} } } }
+
class FilterBySpace(TestCase):
def test_filter_by_space_param_check(self):
client = Mock()
@@ -44,6 +47,7 @@ class FilterBySpace(TestCase):
client = Mock()
ds = 10.0
client.cluster.state.return_value = open_indices
+ client.indices.get_settings.return_value = sized
# Build return value of over 1G in size for each index
client.indices.status.return_value = indices_space
self.assertEqual([], curator.filter_by_space(client, named_indices, disk_space=ds))
@@ -51,6 +55,7 @@ class FilterBySpace(TestCase):
client = Mock()
ds = 2.0
client.cluster.state.return_value = open_indices
+ client.indices.get_settings.return_value = sized
# Build return value of over 1G in size for each index
client.indices.status.return_value = indices_space
self.assertEqual(["index1"], curator.filter_by_space(client, named_indices, disk_space=ds))
@@ -58,6 +63,7 @@ class FilterBySpace(TestCase):
client = Mock()
ds = 2.0
client.cluster.state.return_value = open_indices
+ client.indices.get_settings.return_value = sized
# Build return value of over 1G in size for each index
client.indices.status.return_value = indices_space
self.assertEqual(["index2"], curator.filter_by_space(client, named_indices, disk_space=ds, reverse=False))
@@ -314,3 +320,19 @@ class TestTimestampCheck(TestCase):
utc_now = datetime(2015, 1, 5)
self.assertFalse(curator.timestamp_check(timestamp, timestring=ts,
time_unit=tu, value=v, utc_now=utc_now))
+ def test_timestamp_check_get_cutoff_with_none(self):
+ ts = '%Y.%m.%d'
+ tu = 'days'
+ v = None
+ timestamp = '2015.01.01'
+ utc_now = datetime(2015, 1, 5)
+ self.assertFalse(curator.timestamp_check(timestamp, timestring=ts,
+ time_unit=tu, value=v, utc_now=utc_now))
+ def test_timestamp_check_get_cutoff_with_non_int(self):
+ ts = '%Y.%m.%d'
+ tu = 'days'
+ v = 'foo'
+ timestamp = '2015.01.01'
+ utc_now = datetime(2015, 1, 5)
+ self.assertFalse(curator.timestamp_check(timestamp, timestring=ts,
+ time_unit=tu, value=v, utc_now=utc_now))
diff --git a/test/unit/test_cli_utils.py b/test/unit/test_cli_utils.py
index b3ed634..623e579 100644
--- a/test/unit/test_cli_utils.py
+++ b/test/unit/test_cli_utils.py
@@ -121,3 +121,12 @@ class TestInList(TestCase):
v = ['a', 'b', 'q']
s = ['a', 'b', 'c', 'd']
self.assertEqual(['a', 'b'], curator.in_list(v, s))
+
+class TestGetClient(TestCase):
+ def test_certificate_logic(self):
+ client = Mock()
+ kwargs = { 'use_ssl' : True, 'certificate' : 'mycert.pem' }
+ with self.assertRaises(SystemExit) as cm:
+ curator.get_client(**kwargs)
+ self.assertEqual(sys.stdout.getvalue(),'ERROR: Connection failure.\n')
+ self.assertEqual(cm.exception.code, 1)
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/elasticsearch-curator.git
More information about the Python-modules-commits
mailing list