[Python-modules-commits] [elasticsearch-curator] 01/06: Import elasticsearch-curator_4.1.0.orig.tar.gz
Apollon Oikonomopoulos
apoikos at moszumanska.debian.org
Sat Sep 10 09:12:10 UTC 2016
This is an automated email from the git hooks/post-receive script.
apoikos pushed a commit to branch master
in repository elasticsearch-curator.
commit 546a322ecfac066daf70e100b263f127f2476c78
Author: Apollon Oikonomopoulos <apoikos at debian.org>
Date: Sat Sep 10 12:06:45 2016 +0300
Import elasticsearch-curator_4.1.0.orig.tar.gz
---
.travis.yml | 1 +
README.rst | 5 +-
curator/__init__.py | 3 +-
curator/_version.py | 2 +-
curator/actions.py | 10 +-
curator/cli.py | 93 +++---
curator/defaults/settings.py | 172 ++++-------
curator/indexlist.py | 332 ++++++++++++++-------
curator/repomgrcli.py | 20 +-
curator/snapshotlist.py | 185 +++++++++---
curator/utils.py | 145 +++++++--
curator/validators/__init__.py | 1 +
curator/validators/actions.py | 53 ++++
curator/validators/config_file.py | 50 ++++
curator/validators/filter_elements.py | 120 ++++++++
curator/validators/filters.py | 73 +++++
curator/validators/filtertypes.py | 107 +++++++
curator/validators/options.py | 200 +++++++++++++
curator/validators/schemacheck.py | 69 +++++
docs/Changelog.rst | 31 ++
docs/asciidoc/actions.asciidoc | 2 +
docs/asciidoc/command-line.asciidoc | 3 +
docs/asciidoc/configuration.asciidoc | 84 +++++-
docs/asciidoc/examples.asciidoc | 3 +
docs/asciidoc/faq.asciidoc | 27 ++
docs/asciidoc/filter_elements.asciidoc | 151 ++++++++++
docs/asciidoc/filters.asciidoc | 185 +++++++++---
docs/asciidoc/index.asciidoc | 4 +-
docs/asciidoc/installation.asciidoc | 4 +-
docs/asciidoc/options.asciidoc | 67 +++--
docs/filters.rst | 15 +
docs/utilities.rst | 3 +
requirements.txt | 3 +-
setup.py | 3 +-
test/integration/test_alias.py | 4 +-
test/integration/test_allocation.py | 4 +-
test/integration/test_cli.py | 74 ++++-
test/integration/test_close.py | 12 +-
test/integration/test_create_index.py | 2 +-
test/integration/test_delete_indices.py | 2 +-
test/integration/test_delete_snapshots.py | 4 +-
test/integration/test_envvars.py | 91 ++++++
test/integration/test_forcemerge.py | 2 +-
.../{test_replicas.py => test_integrations.py} | 43 +--
test/integration/test_open.py | 2 +-
test/integration/test_replicas.py | 4 +-
test/integration/test_restore.py | 2 +-
test/integration/test_snapshot.py | 4 +-
test/integration/testvars.py | 57 +++-
test/unit/test_action_close.py | 11 +
test/unit/test_class_index_list.py | 103 ++++++-
test/unit/test_class_snapshot_list.py | 53 +++-
test/unit/test_utils.py | 16 +
test/unit/test_validators.py | 222 ++++++++++++++
test/unit/testvars.py | 7 +
55 files changed, 2453 insertions(+), 492 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index b352c37..850dc52 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,6 +10,7 @@ env:
- ES_VERSION=2.1.1
- ES_VERSION=2.2.2
- ES_VERSION=2.3.5
+ - ES_VERSION=2.4.0
- ES_VERSION=5.0.0-alpha5
os: linux
diff --git a/README.rst b/README.rst
index b23f326..e1baf0b 100644
--- a/README.rst
+++ b/README.rst
@@ -33,7 +33,10 @@ following:
"creation_date"! This implies that the index predates Elasticsearch v1.4.
For safety, this index will be removed from the actionable list.
-
+It is also important to note that Curator 4 requires access to the
+``/_cluster/state/metadata`` endpoint. Forks of Elasticsearch which do not
+support this endpoint (such as AWS ES, see #717) *will not* be able to use
+Curator version 4.
Build Status
------------
diff --git a/curator/__init__.py b/curator/__init__.py
index ff44ee0..b8a2d88 100644
--- a/curator/__init__.py
+++ b/curator/__init__.py
@@ -1,5 +1,6 @@
from .exceptions import *
-from .defaults import settings
+from .defaults import *
+from .validators import *
from .logtools import *
from .utils import *
from .indexlist import IndexList
diff --git a/curator/_version.py b/curator/_version.py
index 9066d7f..fa721b4 100644
--- a/curator/_version.py
+++ b/curator/_version.py
@@ -1 +1 @@
-__version__ = '4.0.6'
+__version__ = '4.1.0'
diff --git a/curator/actions.py b/curator/actions.py
index 92cdd24..bd20529 100644
--- a/curator/actions.py
+++ b/curator/actions.py
@@ -244,8 +244,14 @@ class Close(object):
self.loggit.info(
'Deleting aliases from indices before closing.')
self.loggit.debug('Deleting aliases from: {0}'.format(l))
- self.client.indices.delete_alias(
- index=to_csv(l), name='_all')
+ try:
+ self.client.indices.delete_alias(
+ index=to_csv(l), name='_all')
+ except Exception as e:
+ self.loggit.warn(
+ 'Some indices may not have had aliases. Exception:'
+ ' {0}'.format(e)
+ )
self.client.indices.flush(
index=to_csv(l), ignore_unavailable=True)
self.client.indices.close(
diff --git a/curator/cli.py b/curator/cli.py
index 8da44ab..d2d95a3 100644
--- a/curator/cli.py
+++ b/curator/cli.py
@@ -2,7 +2,9 @@ import os, sys
import yaml
import logging
import click
+from voluptuous import Schema
from .defaults import settings
+from .validators import SchemaCheck, config_file
from .exceptions import *
from .utils import *
from .indexlist import IndexList
@@ -46,18 +48,15 @@ def process_action(client, config, **kwargs):
logger.debug('Configuration dictionary: {0}'.format(config))
logger.debug('kwargs: {0}'.format(kwargs))
action = config['action']
- opts = config['options'] if 'options' in config else {}
+ # This will always have some defaults now, so no need to do the if...
+ # # OLD WAY: opts = config['options'] if 'options' in config else {}
+ opts = config['options']
logger.debug('opts: {0}'.format(opts))
mykwargs = {}
- if action in CLASS_MAP:
- mykwargs = settings.action_defaults()[action]
- action_class = CLASS_MAP[action]
- else:
- raise ConfigurationError(
- 'Unrecognized action: {0}'.format(action))
+ action_class = CLASS_MAP[action]
- # Override some settings...
+ # Add some settings to mykwargs...
if action == 'delete_indices':
mykwargs['master_timeout'] = (
kwargs['master_timeout'] if 'master_timeout' in kwargs else 30)
@@ -65,13 +64,13 @@ def process_action(client, config, **kwargs):
# Setting the operation timeout to the client timeout
mykwargs['timeout'] = (
kwargs['timeout'] if 'timeout' in kwargs else 30)
- logger.debug('MYKWARGS = {0}'.format(mykwargs))
### Update the defaults with whatever came with opts, minus any Nones
mykwargs.update(prune_nones(opts))
logger.debug('Action kwargs: {0}'.format(mykwargs))
- # Verify the args we're going to pass match the action
- verify_args(action, mykwargs)
+ # This is no longer necessary with the config schema validator
+ # # Verify the args we're going to pass match the action
+ # verify_args(action, mykwargs)
### Set up the action ###
if action == 'alias':
@@ -126,15 +125,19 @@ def cli(config, dry_run, action_file):
"""
# Get config from yaml file
yaml_config = get_yaml(config)
- # Get default options and overwrite with any changes
- try:
- yaml_log_opts = prune_nones(yaml_config['logging'])
- log_opts = settings.logs()
- log_opts.update(yaml_log_opts)
- except KeyError:
- # Use the defaults if there is no logging section
- log_opts = settings.logs()
+ # if the file is empty, which is still valid yaml, set as an empty dict
+ yaml_config = {} if not yaml_config else prune_nones(yaml_config)
+ # Voluptuous can't verify the schema of a dict if it doesn't have keys,
+ # so make sure the keys are at least there and are dict()
+ for k in ['client', 'logging']:
+ if k not in yaml_config:
+ yaml_config[k] = {}
+ else:
+ yaml_config[k] = prune_nones(yaml_config[k])
+ config_dict = SchemaCheck(yaml_config, config_file.client(),
+ 'Client Configuration', 'full configuration dictionary').result()
# Set up logging
+ log_opts = config_dict['logging']
loginfo = LogInfo(log_opts)
logging.root.addHandler(loginfo.handler)
logging.root.setLevel(loginfo.numeric_log_level)
@@ -147,18 +150,9 @@ def cli(config, dry_run, action_file):
for handler in logging.root.handlers:
handler.addFilter(Blacklist(bl_entry))
- # Get default client options and overwrite with any changes
- try:
- yaml_client = prune_nones(yaml_config['client'])
- client_args = settings.client()
- client_args.update(yaml_client)
- except KeyError:
- logger.critical(
- 'Unable to read client configuration. '
- 'Please check the configuration file: {0}'.format(config)
- )
- sys.exit(1)
+ client_args = config_dict['client']
test_client_options(client_args)
+ logger.debug('Client and logging options validated.')
# Extract this and save it for later, in case there's no timeout_override.
default_timeout = client_args.pop('timeout')
@@ -166,38 +160,33 @@ def cli(config, dry_run, action_file):
#########################################
### Start working on the actions here ###
#########################################
- actions = get_yaml(action_file)['actions']
+ action_config = get_yaml(action_file)
+ action_dict = validate_actions(action_config)
+ actions = action_dict['actions']
logger.debug('Full list of actions: {0}'.format(actions))
action_keys = sorted(list(actions.keys()))
for idx in action_keys:
- if 'action' in actions[idx] and actions[idx]['action'] is not None:
- action = actions[idx]['action'].lower()
- else:
- raise MissingArgument('No value for "action" provided')
- logger.info('Action #{0}: {1}'.format(idx, action))
- if not 'options' in actions[idx] or \
- type(actions[idx]['options']) is not type(dict()):
- actions[idx]['options'] = settings.options()
- # Assign and remove these keys from the options as the action will
- # raise an exception if they are passed as kwargs
- action_disabled = actions[idx]['options'].pop('disable_action', False)
+ action = actions[idx]['action']
+ action_disabled = actions[idx]['options'].pop('disable_action')
+ logger.debug('action_disabled = {0}'.format(action_disabled))
continue_if_exception = (
- actions[idx]['options'].pop('continue_if_exception', False))
- timeout_override = actions[idx]['options'].pop('timeout_override', None)
- ignore_empty_list = actions[idx]['options'].pop(
- 'ignore_empty_list', None)
+ actions[idx]['options'].pop('continue_if_exception'))
logger.debug(
'continue_if_exception = {0}'.format(continue_if_exception))
+ timeout_override = actions[idx]['options'].pop('timeout_override')
logger.debug('timeout_override = {0}'.format(timeout_override))
+ ignore_empty_list = actions[idx]['options'].pop('ignore_empty_list')
+ logger.debug('ignore_empty_list = {0}'.format(ignore_empty_list))
### Skip to next action if 'disabled'
if action_disabled:
logger.info(
- 'Action "{0}" not performed because "disable_action" is set to '
- 'True'.format(action)
+ 'Action ID: {0}: "{1}" not performed because "disable_action" '
+ 'is set to True'.format(idx, action)
)
continue
-
+ else:
+ logger.info('Preparing Action ID: {0}, "{1}"'.format(idx, action))
# Override the timeout, if specified, otherwise use the default.
if type(timeout_override) == type(int()):
client_args['timeout'] = timeout_override
@@ -218,8 +207,8 @@ def cli(config, dry_run, action_file):
### Process the action ###
##########################
try:
- logger.debug('TRY: actions: {0} kwargs: '
- '{1}'.format(actions[idx], kwargs)
+ logger.info('Trying Action ID: {0}, "{1}": '
+ '{2}'.format(idx, action, actions[idx]['description'])
)
process_action(client, actions[idx], **kwargs)
except Exception as e:
@@ -249,5 +238,5 @@ def cli(config, dry_run, action_file):
)
else:
sys.exit(1)
- logger.info('Action #{0}: completed'.format(idx))
+ logger.info('Action ID: {0}, "{1}" completed.'.format(idx, action))
logger.info('Job completed.')
diff --git a/curator/defaults/settings.py b/curator/defaults/settings.py
index c767f08..cc53d13 100644
--- a/curator/defaults/settings.py
+++ b/curator/defaults/settings.py
@@ -1,4 +1,5 @@
import os
+from voluptuous import *
# Elasticsearch versions supported
def version_max():
@@ -6,43 +7,11 @@ def version_max():
def version_min():
return (2, 0, 0)
+# Default Config file location
def config_file():
return os.path.join(os.path.expanduser('~'), '.curator', 'curator.yml')
-def client():
- return {
- 'hosts': '127.0.0.1',
- 'port': 9200,
- 'url_prefix': '',
- 'http_auth': None,
- 'use_ssl': False,
- 'certificate': None,
- 'client_cert': None,
- 'client_key': None,
- 'aws_key': None,
- 'aws_secret_key': None,
- 'aws_region': None,
- 'ssl_no_validate': False,
- 'timeout': 30,
- 'master_only': False,
- }
-
-def logs():
- return {
- 'loglevel': 'INFO',
- 'logfile': None,
- 'logformat': 'default',
- 'blacklist': ['elasticsearch', 'urllib3'],
- }
-
-def options():
- return {
- 'ignore_empty_list': False,
- 'timeout_override': None,
- 'continue_if_exception': False,
- 'disable_action': False,
- }
-
+# Default filter patterns (regular expressions)
def regex_map():
return {
'timestring': r'^.*{0}.*$',
@@ -65,95 +34,54 @@ def date_regex():
'j' : '3',
}
-def action_defaults():
- return {
- 'alias' : {
- 'name' : None,
- 'extra_settings' : {},
- },
- 'allocation' : {
- 'key' : None,
- 'value' : None,
- 'allocation_type' : 'require',
- 'wait_for_completion' : False,
- 'timeout' : 30,
- },
- 'close' : { 'delete_aliases' : False },
- 'create_index' : {
- 'name' : None,
- 'extra_settings' : {},
- },
- 'delete_indices' : { 'master_timeout' : 30 },
- 'delete_snapshots' : {
- 'repository' : None,
- 'retry_interval' : 120,
- 'retry_count' : 3,
- },
- 'forcemerge' : {
- 'delay' : 0,
- 'max_num_segments' : 2,
- },
- 'open' : {},
- 'replicas' : {
- 'count' : None,
- 'wait_for_completion' : False,
- 'timeout' : 30,
- },
- 'restore' : {
- 'repository' : None,
- 'name' : None,
- 'indices' : None,
- 'include_aliases' : False,
- 'ignore_unavailable' : False,
- 'include_global_state' : True,
- 'partial' : False,
- 'rename_pattern' : None,
- 'rename_replacement' : None,
- 'extra_settings' : {},
- 'wait_for_completion' : True,
- 'skip_repo_fs_check' : False,
- },
- 'snapshot' : {
- 'repository' : None,
- 'name' : 'curator-%Y%m%d%H%M%S',
- 'ignore_unavailable' : False,
- 'include_global_state' : True,
- 'partial' : False,
- 'wait_for_completion' : True,
- 'skip_repo_fs_check' : False,
- },
- }
+# Actions
+def index_actions():
+ return [
+ 'alias',
+ 'allocation',
+ 'close',
+ 'create_index',
+ 'delete_indices',
+ 'forcemerge',
+ 'open',
+ 'replicas',
+ 'snapshot',
+ ]
-def index_filter():
- return {
- 'age': {
- 'source':'name', 'direction':None, 'timestring':None, 'unit':None,
- 'unit_count':None, 'field':None, 'stats_result':'min_value',
- 'epoch':None, 'exclude':False
- },
- 'allocated': {
- 'key':None, 'value':None, 'allocation_type':'require', 'exclude':True
- },
- 'closed': {'exclude':True},
- 'forcemerged': {'max_num_segments':None, 'exclude':True},
- 'kibana': {'exclude':True},
- 'none': {},
- 'opened': {'exclude':True},
- 'pattern': {'kind':None, 'value':None, 'exclude':False},
- 'space': {
- 'disk_space':None, 'reverse':True, 'use_age':False,
- 'source':'creation_date', 'timestring':None, 'field':None,
- 'stats_result':'min_value', 'exclude':False,
- },
- }
+def snapshot_actions():
+ return [ 'delete_snapshots', 'restore' ]
+
+def all_actions():
+ return sorted(index_actions() + snapshot_actions())
+
+def index_filtertypes():
+ return [
+ 'alias',
+ 'allocated',
+ 'age',
+ 'closed',
+ 'count',
+ 'forcemerged',
+ 'kibana',
+ 'none',
+ 'opened',
+ 'pattern',
+ 'space',
+ ]
+
+def snapshot_filtertypes():
+ return ['age', 'count', 'none', 'pattern', 'state']
-def snapshot_filter():
+def all_filtertypes():
+ return sorted(list(set(index_filtertypes() + snapshot_filtertypes())))
+
+def default_options():
return {
- 'age': {
- 'source':'creation_date', 'direction':None, 'timestring':None,
- 'unit':None, 'unit_count':None, 'epoch':None, 'exclude':False
- },
- 'none': {},
- 'pattern': {'kind':None, 'value':None, 'exclude':False},
- 'state': {'state':'SUCCESS', 'exclude':False}
+ 'continue_if_exception': False,
+ 'disable_action': False,
+ 'ignore_empty_list': False,
+ 'timeout_override': None,
}
+
+def default_filters():
+ return { 'filters' : [{ 'filtertype' : 'none' }] }
diff --git a/curator/indexlist.py b/curator/indexlist.py
index 180a445..e47a37f 100644
--- a/curator/indexlist.py
+++ b/curator/indexlist.py
@@ -2,11 +2,12 @@ from datetime import timedelta, datetime, date
import time
import re
import logging
+import elasticsearch
from .defaults import settings
+from .validators import SchemaCheck, filters
from .exceptions import *
from .utils import *
-
class IndexList(object):
def __init__(self, client):
verify_client_object(client)
@@ -75,7 +76,8 @@ class IndexList(object):
Ensure that `index` is a key in `index_info`. If not, create a
sub-dictionary structure under that key.
"""
- self.loggit.debug('Building index info dictionary')
+ self.loggit.debug(
+ 'Building preliminary index metadata for {0}'.format(index))
if not index in self.index_info:
self.index_info[index] = {
"age" : {},
@@ -89,9 +91,11 @@ class IndexList(object):
def __map_method(self, ft):
methods = {
+ 'alias': self.filter_by_alias,
'age': self.filter_by_age,
'allocated': self.filter_allocated,
'closed': self.filter_closed,
+ 'count': self.filter_by_count,
'forcemerged': self.filter_forceMerged,
'kibana': self.filter_kibana,
'none': self.filter_none,
@@ -262,6 +266,84 @@ class IndexList(object):
'"{1}"'.format(field, index)
)
+ def _calculate_ages(self, source=None, timestring=None, field=None,
+ stats_result=None
+ ):
+ """
+ This method initiates index age calculation based on the given
+ parameters. Exceptions are raised when they are improperly configured.
+
+ Set instance variable `age_keyfield` for use later, if needed.
+
+ :arg source: Source of index age. Can be one of 'name', 'creation_date',
+ or 'field_stats'
+ :arg timestring: An strftime string to match the datestamp in an index
+ name. Only used for index filtering by ``name``.
+ :arg field: A timestamp field name. Only used for ``field_stats`` based
+ calculations.
+ :arg stats_result: Either `min_value` or `max_value`. Only used in
+ conjunction with `source`=``field_stats`` to choose whether to
+ reference the minimum or maximum result value.
+ """
+ self.age_keyfield = source
+ if source == 'name':
+ if not timestring:
+ raise MissingArgument(
+ 'source "name" requires the "timestring" keyword argument'
+ )
+ self._get_name_based_ages(timestring)
+ elif source == 'creation_date':
+ # Nothing to do here as this comes from `get_metadata` in __init__
+ pass
+ elif source == 'field_stats':
+ if not field:
+ raise MissingArgument(
+ 'source "field_stats" requires the "field" keyword argument'
+ )
+ if stats_result not in ['min_value', 'max_value']:
+ raise ValueError(
+ 'Invalid value for "stats_result": {0}'.format(stats_result)
+ )
+ self.age_keyfield = stats_result
+ self._get_field_stats_dates(field=field)
+ else:
+ raise ValueError(
+ 'Invalid source: {0}. '
+ 'Must be one of "name", '
+ '"creation_date", "field_stats".'.format(source)
+ )
+
+ def _sort_by_age(self, index_list, reverse=True):
+ """
+ Take a list of indices and sort them by date.
+
+ By default, the youngest are first with `reverse=True`, but the oldest
+ can be first by setting `reverse=False`
+ """
+ # Do the age-based sorting here.
+ # First, build an temporary dictionary with just index and age
+ # as the key and value, respectively
+ temp = {}
+ for index in index_list:
+ if self.age_keyfield in self.index_info[index]['age']:
+ temp[index] = self.index_info[index]['age'][self.age_keyfield]
+ else:
+ msg = (
+ '{0} does not have age key "{1}" in IndexList '
+ ' metadata'.format(index, self.age_keyfield)
+ )
+ self.__excludify(True, True, index, msg)
+
+ # If reverse is True, this will sort so the youngest indices are first.
+ # However, if you want oldest first, set reverse to False.
+ # Effectively, this should set us up to act on everything older than
+ # meets the other set criteria.
+ # It starts as a tuple, but then becomes a list.
+ sorted_tuple = (
+ sorted(temp.items(), key=lambda k: k[1], reverse=reverse)
+ )
+ return [x[0] for x in sorted_tuple]
+
def filter_by_regex(self, kind=None, value=None, exclude=False):
"""
Match indices by regular expression (pattern).
@@ -338,48 +420,23 @@ class IndexList(object):
self.loggit.debug('Filtering indices by age')
# Get timestamp point of reference, PoR
PoR = get_point_of_reference(unit, unit_count, epoch)
- keyfield = source
if not direction:
raise MissingArgument('Must provide a value for "direction"')
if direction not in ['older', 'younger']:
raise ValueError(
'Invalid value for "direction": {0}'.format(direction)
)
- if source == 'name':
- if not timestring:
- raise MissingArgument(
- 'source "name" requires the "timestring" keyword argument'
- )
- self._get_name_based_ages(timestring)
- elif source == 'creation_date':
- # Nothing to do here as this comes from `get_metadata` in __init__
- pass
- elif source == 'field_stats':
- if not field:
- raise MissingArgument(
- 'source "field_stats" requires the "field" keyword argument'
- )
- if stats_result not in ['min_value', 'max_value']:
- raise ValueError(
- 'Invalid value for "stats_result": {0}'.format(stats_result)
- )
- keyfield = stats_result
- self._get_field_stats_dates(field=field)
- else:
- raise ValueError(
- 'Invalid source: {0}. '
- 'Must be one of "name", '
- '"creation_date", "field_stats".'.format(source)
- )
-
+ self._calculate_ages(
+ source=source, timestring=timestring, field=field,
+ stats_result=stats_result
+ )
for index in self.working_list():
-
try:
msg = (
'Index "{0}" age ({1}), direction: "{2}", point of '
'reference, ({3})'.format(
index,
- int(self.index_info[index]['age'][keyfield]),
+ int(self.index_info[index]['age'][self.age_keyfield]),
direction,
PoR
)
@@ -387,9 +444,9 @@ class IndexList(object):
# Because time adds to epoch, smaller numbers are actually older
# timestamps.
if direction == 'older':
- agetest = self.index_info[index]['age'][keyfield] < PoR
+ agetest = self.index_info[index]['age'][self.age_keyfield] < PoR
else:
- agetest = self.index_info[index]['age'][keyfield] > PoR
+ agetest = self.index_info[index]['age'][self.age_keyfield] > PoR
self.__excludify(agetest, exclude, index, msg)
except KeyError:
self.loggit.debug(
@@ -458,54 +515,12 @@ class IndexList(object):
working_list = self.working_list()
if use_age:
- keyfield = source
- if source not in ['creation_date', 'name', 'field_stats']:
- raise ValueError(
- 'Invalid value for "source": {0}'.format(source)
- )
- if source == 'field_stats':
- if not field:
- raise MissingArgument(
- 'No value for "field" provided. "field" is required '
- 'with source "field_stats"'
- )
- if stats_result not in ['min_value', 'max_value']:
- raise ConfigurationError(
- 'Incorrect value for "stats_result" provided: {0}. '
- 'Must be either "min_value" or '
- '"max_value"'.format(stats_result)
- )
- keyfield = stats_result
- self._get_field_stats_dates(field=field)
- if source == 'name':
- if not timestring:
- raise MissingArgument(
- 'No value for "timestring" provided. "timestring" is '
- 'required with source "name"'
- )
- self._get_name_based_ages(timestring)
-
- # Do the age-based sorting here.
- # First, build an intermediate dictionary with just index and age
- # as the key and value, respectively
- intermediate = {}
- for index in working_list:
- if keyfield in self.index_info[index]['age']:
- intermediate[index] = self.index_info[index]['age'][keyfield]
- else:
- msg = (
- '{0} does not have age key "{1}" in IndexList '
- ' metadata'.format(index, keyfield)
- )
- self.__excludify(True, True, index, msg)
-
- # This will sort the indices the youngest first. Effectively, this
- # should set us up to delete everything older than fits into
- # `disk_space`. It starts as a tuple, but then becomes a list.
- sorted_tuple = (
- sorted(intermediate.items(), key=lambda k: k[1], reverse=True)
+ self._calculate_ages(
+ source=source, timestring=timestring, field=field,
+ stats_result=stats_result
)
- sorted_indices = [x[0] for x in sorted_tuple]
+ # Using default value of reverse=True in self._sort_by_age()
+ sorted_indices = self._sort_by_age(working_list)
else:
# Default to sorting by index name
@@ -662,6 +677,125 @@ class IndexList(object):
def filter_none(self):
self.loggit.debug('"None" filter selected. No filtering will be done.')
+ def filter_by_alias(self, aliases=None, exclude=False):
+ """
+ Match indices which are associated with the alias identified by `name`
+
+ :arg aliases: A list of alias names.
+ :type aliases: list
+ :arg exclude: If `exclude` is `True`, this filter will remove matching
+ indices from `indices`. If `exclude` is `False`, then only matching
+ indices will be kept in `indices`.
+ Default is `False`
+ """
+ self.loggit.debug(
+ 'Filtering indices matching aliases: "{0}"'.format(aliases))
+ if not aliases:
+ raise MissingArgument('No value for "aliases" provided')
+ aliases = ensure_list(aliases)
+ self.empty_list_check()
+ index_lists = chunk_index_list(self.indices)
+ for l in index_lists:
+ try:
+ # get_alias will either return {} or a NotFoundError.
+ has_alias = list(self.client.indices.get_alias(
+ index=to_csv(l),
+ name=to_csv(aliases)
+ ).keys())
+ self.loggit.debug('has_alias: {0}'.format(has_alias))
+ except elasticsearch.exceptions.NotFoundError:
+ # if we see the NotFoundError, we need to set working_list to {}
+ has_alias = []
+ for index in l:
+ if index in has_alias:
+ isOrNot = 'is'
+ condition = True
+ else:
+ isOrNot = 'is not'
+ condition = False
+ msg = (
+ '{0} {1} associated with aliases: {2}'.format(
+ index, isOrNot, aliases
+ )
+ )
+ self.__excludify(condition, exclude, index, msg)
+
+ def filter_by_count(
+ self, count=None, reverse=True, use_age=False,
+ source='creation_date', timestring=None, field=None,
+ stats_result='min_value', exclude=True):
+ """
+ Remove indices from the actionable list beyond the number `count`,
+ sorted reverse-alphabetically by default. If you set `reverse` to
+ `False`, it will be sorted alphabetically.
+
+ The default is usually what you will want. If only one kind of index is
+ provided--for example, indices matching ``logstash-%Y.%m.%d``--then
+ reverse alphabetical sorting will mean the oldest will remain in the
+ list, because lower numbers in the dates mean older indices.
+
+ By setting `reverse` to `False`, then ``index3`` will be deleted before
+ ``index2``, which will be deleted before ``index1``
+
+ `use_age` allows ordering indices by age. Age is determined by the index
+ creation date by default, but you can specify an `source` of ``name``,
+ ``max_value``, or ``min_value``. The ``name`` `source` requires the
+ timestring argument.
+
+ :arg count: Filter indices beyond `count`.
+ :arg reverse: The filtering direction. (default: `True`).
+ :arg use_age: Sort indices by age. ``source`` is required in this
+ case.
+ :arg source: Source of index age. Can be one of ``name``,
+ ``creation_date``, or ``field_stats``. Default: ``creation_date``
+ :arg timestring: An strftime string to match the datestamp in an index
+ name. Only used if `source` ``name`` is selected.
+ :arg field: A timestamp field name. Only used if `source`
+ ``field_stats`` is selected.
+ :arg stats_result: Either `min_value` or `max_value`. Only used if
+ `source` ``field_stats`` is selected. It determines whether to
+ reference the minimum or maximum value of `field` in each index.
+ :arg exclude: If `exclude` is `True`, this filter will remove matching
+ indices from `indices`. If `exclude` is `False`, then only matching
+ indices will be kept in `indices`.
+ Default is `True`
+ """
+ self.loggit.debug('Filtering indices by count')
+ if not count:
+ raise MissingArgument('No value for "count" provided')
+
+ # Create a copy-by-value working list
+ working_list = self.working_list()
+
+ if use_age:
+ if source is not 'name':
+ self.loggit.warn(
+ 'Cannot get age information from closed indices unless '
+ 'source="name". Omitting any closed indices.'
+ )
+ self.filter_closed()
+ self._calculate_ages(
+ source=source, timestring=timestring, field=field,
+ stats_result=stats_result
+ )
+ # Using default value of reverse=True in self._sort_by_age()
+ sorted_indices = self._sort_by_age(working_list, reverse=reverse)
+
+ else:
+ # Default to sorting by index name
+ sorted_indices = sorted(working_list, reverse=reverse)
+
+ idx = 1
+ for index in sorted_indices:
+ msg = (
+ '{0} is {1} of specified count of {2}.'.format(
+ index, idx, count
+ )
+ )
+ condition = True if idx <= count else False
+ self.__excludify(condition, exclude, index, msg)
+ idx += 1
+
def iterate_filters(self, filter_dict):
"""
Iterate over the filters defined in `config` and execute them.
@@ -691,35 +825,25 @@ class IndexList(object):
self.loggit.debug('All filters: {0}'.format(filter_dict['filters']))
for f in filter_dict['filters']:
self.loggit.debug('Top of the loop: {0}'.format(self.indices))
- logger.debug('Un-parsed filter args: {0}'.format(f))
- f_args = None
+ self.loggit.debug('Un-parsed filter args: {0}'.format(f))
# Make sure we got at least this much in the configuration
- if not 'filtertype' in f:
- raise ConfigurationError(
- 'No "filtertype" in filter definition.'
+ self.loggit.debug('Parsed filter args: {0}'.format(
+ SchemaCheck(
+ f,
+ filters.structure(),
+ 'filter',
+ 'IndexList.iterate_filters'
+ ).result()
)
- try:
- ft = f['filtertype'].lower()
- except Exception as e:
- raise ValueError(
- 'Invalid value for "filtertype": '
- '{0}'.format(f['filtertype'])
- )
- try:
- f_args = settings.index_filter()[ft]
- method = self.__map_method(ft)
- except:
- raise ConfigurationError(
- 'Unrecognized filtertype: {0}'.format(ft))
- # Remove key 'filtertype' from dictionary 'f'
+ )
+ method = self.__map_method(f['filtertype'])
del f['filtertype']
# If it's a filtertype with arguments, update the defaults with the
# provided settings.
- if f_args:
- f_args.update(prune_nones(f))
- logger.debug('Filter args: {0}'.format(f_args))
+ if f:
+ logger.debug('Filter args: {0}'.format(f))
logger.debug('Pre-instance: {0}'.format(self.indices))
- method(**f_args)
+ method(**f)
logger.debug('Post-instance: {0}'.format(self.indices))
else:
# Otherwise, it's a settingless filter.
diff --git a/curator/repomgrcli.py b/curator/repomgrcli.py
index 9aaf242..5ced77e 100644
--- a/curator/repomgrcli.py
+++ b/curator/repomgrcli.py
@@ -95,24 +95,22 @@ def s3(
@click.group()
@click.option(
- '--host', help='Elasticsearch host.', default=settings.client()['hosts'])
+ '--host', help='Elasticsearch host.', default='127.0.0.1')
@click.option(
- '--url_prefix', help='Elasticsearch http url prefix.',
- default=settings.client()['url_prefix']
-)
- at click.option('--port', help='Elasticsearch port.', default=settings.client()['port'], type=int)
- at click.option('--use_ssl', help='Connect to Elasticsearch through SSL.', is_flag=True, default=settings.client()['use_ssl'])
+ '--url_prefix', help='Elasticsearch http url prefix.',default='')
+ at click.option('--port', help='Elasticsearch port.', default=9200, type=int)
+ at click.option('--use_ssl', help='Connect to Elasticsearch through SSL.', is_flag=True)
@click.option('--certificate', help='Path to certificate to use for SSL validation. (OPTIONAL)', type=str, default=None)
@click.option('--client-cert', help='Path to file containing SSL certificate for client auth. (OPTIONAL)', type=str, default=None)
@click.option('--client-key', help='Path to file containing SSL key for client auth. (OPTIONAL)', type=str, default=None)
@click.option('--ssl-no-validate', help='Do not validate server\'s SSL certificate', is_flag=True)
- at click.option('--http_auth', help='Use Basic Authentication ex: user:pass', default=settings.client()['http_auth'])
- at click.option('--timeout', help='Connection timeout in seconds.', default=settings.client()['timeout'], type=int)
+ at click.option('--http_auth', help='Use Basic Authentication ex: user:pass', default='')
+ at click.option('--timeout', help='Connection timeout in seconds.', default=30, type=int)
@click.option('--master-only', is_flag=True, help='Only operate on elected master node.')
@click.option('--debug', is_flag=True, help='Debug mode')
- at click.option('--loglevel', help='Log level', default=settings.logs()['loglevel'])
- at click.option('--logfile', help='log file', default=settings.logs()['logfile'])
- at click.option('--logformat', help='Log output format [default|logstash].', default=settings.logs()['logformat'])
+ at click.option('--loglevel', help='Log level', default='INFO')
+ at click.option('--logfile', help='log file', default=None)
+ at click.option('--logformat', help='Log output format [default|logstash].', default='default')
@click.version_option(version=__version__)
@click.pass_context
def repo_mgr_cli(
diff --git a/curator/snapshotlist.py b/curator/snapshotlist.py
index 051bac3..b80fbbe 100644
--- a/curator/snapshotlist.py
+++ b/curator/snapshotlist.py
@@ -3,6 +3,7 @@ import time
import re
import logging
from .defaults import settings
+from .validators import SchemaCheck, filters
from .exceptions import *
from .utils import *
@@ -123,6 +124,63 @@ class SnapshotList(object):
else:
self.snapshot_info[snapshot]['age_by_name'] = None
+ def _calculate_ages(self, source='creation_date', timestring=None):
+ """
+ This method initiates snapshot age calculation based on the given
+ parameters. Exceptions are raised when they are improperly configured.
+
+ Set instance variable `age_keyfield` for use later, if needed.
+
+ :arg source: Source of snapshot age. Can be 'name' or 'creation_date'.
+ :arg timestring: An strftime string to match the datestamp in an
+ snapshot name. Only used if ``source`` is ``name``.
+ """
+ if source == 'name':
+ self.age_keyfield = 'age_by_name'
+ if not timestring:
+ raise MissingArgument(
... 3221 lines suppressed ...
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/elasticsearch-curator.git
More information about the Python-modules-commits
mailing list