[Pkg-nagios-changes] [SCM] UNNAMED PROJECT branch, debian/master, updated. 810edbdd3feedbfe37f4a65bee50b57b2f60fa2a
Gerhard Lausser
gerhard.lausser at consol.de
Tue Feb 28 22:15:41 UTC 2012
The following commit has been merged in the debian/master branch:
commit 41a218e4aafa1f17a4b5ae93b2a540298c902390
Author: Gerhard Lausser <gerhard.lausser at consol.de>
Date: Sun Jan 22 21:27:40 2012 +0100
More bugfixes in the MongoDB log storage and a better test
diff --git a/shinken/modules/logstore_mongodb.py b/shinken/modules/logstore_mongodb.py
index f24f46b..66c4f08 100644
--- a/shinken/modules/logstore_mongodb.py
+++ b/shinken/modules/logstore_mongodb.py
@@ -53,6 +53,7 @@ class LiveStatusLogStoreMongoDB(BaseModule):
# mongodb://host1,host2,host3/?safe=true;w=2;wtimeoutMS=2000
self.mongodb_uri = getattr(modconf, 'mongodb_uri', None)
self.database = getattr(modconf, 'database', 'logs')
+ self.collection = getattr(modconf, 'collection', 'logs')
self.use_aggressive_sql = True
max_logs_age = getattr(modconf, 'max_logs_age', '365')
maxmatch = re.match(r'^(\d+)([dwm]*)$', max_logs_age)
@@ -92,7 +93,7 @@ class LiveStatusLogStoreMongoDB(BaseModule):
try:
self.conn = pymongo.Connection(self.mongodb_uri, fsync=True)
self.db = self.conn[self.database]
-
+ self.db[self.collection].ensure_index([('time', pymongo.ASCENDING), ('lineno', pymongo.ASCENDING)], name='time_idx')
self.is_connected = True
except AutoReconnect, exp:
# now what, ha?
@@ -122,7 +123,10 @@ class LiveStatusLogStoreMongoDB(BaseModule):
print "Unexpected error:", exp
try:
if logline.logclass != LOGCLASS_INVALID:
- self.db.logs.insert(values, safe=True)
+ self.db[self.collection].insert(values, safe=True)
+ else:
+ print "This line is invalid", line
+
except Exception, exp:
print "An error occurred:", exp
print "DATABASE ERROR!!!!!!!!!!!!!!!!!"
@@ -162,15 +166,15 @@ class LiveStatusLogStoreMongoDB(BaseModule):
# We can apply the filterstack here as well. we have columns and filtercolumns.
# the only additional step is to enrich log lines with host/service-attributes
# A timerange can be useful for a faster preselection of lines
- filter_element = eval(mongo_filter)
- print "mongo filter iis", type(filter_element)
- print "mongo filter iis", filter_element
+ filter_element = eval('{ '+mongo_filter+' }')
+ print "mongo filter is", filter_element
dbresult = []
columns = ['logobject', 'attempt', 'logclass', 'command_name', 'comment', 'contact_name', 'host_name', 'lineno', 'message', 'options', 'plugin_output', 'service_description', 'state', 'state_type', 'time', 'type']
if not self.is_connected:
print "sorry, not connected"
else:
- dbresult = [Logline([(c, ) for c in columns], [x[col] for col in columns]) for x in self.db.logs.find(filter_element)]
+ dbresult = [Logline([(c, ) for c in columns], [x[col] for col in columns]) for x in self.db[self.collection].find(filter_element).sort([(u'time', pymongo.ASCENDING), (u'lineno', pymongo.ASCENDING)])]
+ print "i count", self.db[self.collection].find(filter_element).count()
return dbresult
def make_mongo_filter(self, operator, attribute, reference):
@@ -180,8 +184,9 @@ class LiveStatusLogStoreMongoDB(BaseModule):
good_attributes = ['time', 'attempt', 'class', 'command_name', 'comment', 'contact_name', 'host_name', 'plugin_output', 'service_description', 'state', 'state_type', 'type']
good_operators = ['=', '!=']
# put strings in '' for the query
- if attribute in ['command_name', 'comment', 'contact_name', 'host_name', 'plugin_output', 'service_description', 'state_type', 'type']:
- attribute = "'%s'" % attribute
+ string_attributes = ['command_name', 'comment', 'contact_name', 'host_name', 'plugin_output', 'service_description', 'state_type', 'type']
+ if attribute in string_attributes:
+ reference = "'%s'" % reference
def eq_filter():
if reference == '':
@@ -202,9 +207,9 @@ class LiveStatusLogStoreMongoDB(BaseModule):
def le_filter():
return '\'%s\' : { \'$lte\' : %s }' % (attribute, reference)
def match_filter():
- return '\'%s\' : { \'$regex\' : \'%s\' }' % (attribute, reference)
+ return '\'%s\' : { \'$regex\' : %s }' % (attribute, reference)
def no_filter():
- return '{}'
+ return '\'%s\' : { \'$exists\' : true }' % (attribute,)
if attribute not in good_attributes:
return no_filter
if operator == '=':
@@ -256,7 +261,7 @@ class LiveStatusMongoStack(LiveStatusStack):
# Make a combined anded function
# Put it on the stack
print "filter is", filters
- and_clause = lambda: '{\'$and\' : [%s]}' % ', '.join('{ ' + x() + ' }' for x in filters)
+ and_clause = lambda: '\'$and\' : [%s]' % ', '.join('{ ' + x() + ' }' for x in filters)
print "and_elements", and_clause
self.put_stack(and_clause)
@@ -266,7 +271,7 @@ class LiveStatusMongoStack(LiveStatusStack):
filters = []
for _ in range(num):
filters.append(self.get_stack())
- or_clause = lambda: '{\'$or\' : [%s]}' % ', '.join('{ ' + x() + ' }' for x in filters)
+ or_clause = lambda: '\'$or\' : [%s]' % ', '.join('{ ' + x() + ' }' for x in filters)
print "or_elements", or_clause
self.put_stack(or_clause)
diff --git a/shinken/modules/logstore_sqlite.py b/shinken/modules/logstore_sqlite.py
index f22996a..1e13776 100644
--- a/shinken/modules/logstore_sqlite.py
+++ b/shinken/modules/logstore_sqlite.py
@@ -380,9 +380,8 @@ class LiveStatusLogStoreSqlite(BaseModule):
#FIXME need access to this#self.livestatus.count_event('log_message')
def add_filter(self, operator, attribute, reference):
- print "isql add", operator, attribute, reference
if attribute == 'time':
- #self.sql_time_filter_stack.put_stack(self.make_sql_filter(operator, attribute, reference))
+ self.sql_time_filter_stack.put_stack(self.make_sql_filter(operator, attribute, reference))
pass
self.sql_filter_stack.put_stack(self.make_sql_filter(operator, attribute, reference))
@@ -526,7 +525,6 @@ class LiveStatusSqlStack(LiveStatusStack):
filters = []
for _ in range(num):
filters.append(self.get_stack())
- print "now i and", [x() for x in filters]
# Take from the stack:
# Make a combined anded function
# Put it on the stack
diff --git a/test/test_livestatus_db.py b/test/test_livestatus_db.py
index 5150a42..b79c3db 100755
--- a/test/test_livestatus_db.py
+++ b/test/test_livestatus_db.py
@@ -623,23 +623,6 @@ OutputFormat: json"""
pyresponse = eval(response)
print "number of records", len(pyresponse)
print "should be", should_be
- numlogs = self.livestatus_broker.db.execute("SELECT min(time), max(time) FROM logs")
- print starttime, endtime, numlogs
- self.livestatus_broker.livestatus.use_aggressive_sql = True
- print "aggrosql", self.livestatus_broker.livestatus.use_aggressive_sql
- response2, keepalive = self.livestatus_broker.livestatus.handle_request(request)
- self.assert_(response2 == response)
- print "aggrosql", self.livestatus_broker.livestatus.use_aggressive_sql
- response2, keepalive = self.livestatus_broker.livestatus.handle_request(request)
- self.assert_(response2 == response)
- self.livestatus_broker.livestatus.use_aggressive_sql = False
- print "aggrosql", self.livestatus_broker.livestatus.use_aggressive_sql
- response2, keepalive = self.livestatus_broker.livestatus.handle_request(request)
- self.assert_(response2 == response)
- print "aggrosql", self.livestatus_broker.livestatus.use_aggressive_sql
- response2, keepalive = self.livestatus_broker.livestatus.handle_request(request)
- self.assert_(response2 == response)
- # back to fake time for the other tests can run faster
time.time = fake_time_time
time.sleep = fake_time_sleep
diff --git a/test/test_livestatus_mongodb.py b/test/test_livestatus_mongodb.py
index 0ec2400..5b1da9c 100755
--- a/test/test_livestatus_mongodb.py
+++ b/test/test_livestatus_mongodb.py
@@ -279,8 +279,8 @@ class TestConfigBig(TestConfig):
host.__class__.use_aggressive_host_checking = 1
- def xinit_livestatus(self):
- self.livelogs = 'tmp/livelogs.db' + "wrumm"
+ def init_livestatus(self):
+ self.livelogs = "bigbigbig"
modconf = Module({'module_name' : 'LiveStatus',
'module_type' : 'livestatus',
'port' : str(50000 + os.getpid()),
@@ -292,6 +292,7 @@ class TestConfigBig(TestConfig):
dbmodconf = Module({'module_name' : 'LogStore',
'module_type' : 'logstore_mongodb',
+ 'database' : 'bigbigbig',
'mongodb_uri' : "mongodb://127.0.0.1:27017",
})
modconf.modules = [dbmodconf]
@@ -346,7 +347,11 @@ class TestConfigBig(TestConfig):
starttime = time.time()
num_log_broks = 0
- if True:
+ try:
+ numlogs = self.livestatus_broker.db.conn.bigbigbig.find().count()
+ except Exception:
+ numlogs = 0
+ if numlogs == 0:
# run silently
old_stdout = sys.stdout
sys.stdout = open(os.devnull, "w")
@@ -360,6 +365,8 @@ class TestConfigBig(TestConfig):
num_log_broks += self.count_log_broks()
self.update_broker()
should_be = 0
+ should_be_huhu = 0
+ huhuhus = []
#for i in xrange(3600 * 24 * 7):
for i in xrange(10000):
if i % 1000 == 0:
@@ -370,9 +377,11 @@ class TestConfigBig(TestConfig):
[test_ok_01, 2, "CRIT"],
[test_ok_04, 3, "UNKN"],
[test_ok_16, 1, "WARN"],
- [test_ok_99, 2, "CRIT"],
+ [test_ok_99, 2, "HUHU"+str(i)],
])
should_be += 3
+ should_be_huhu += 3
+ huhuhus.append(i)
time.sleep(62)
if i % 399 == 0:
self.scheduler_loop(1, [
@@ -423,11 +432,15 @@ class TestConfigBig(TestConfig):
sys.stdout = old_stdout
self.livestatus_broker.db.commit()
else:
- should_be = numlogs[0][0]
- xxx = self.livestatus_broker.db.execute("SELECT min(time), max(time) FROM logs")
- print xxx
- starttime, endtime = [self.livestatus_broker.db.execute("SELECT min(time), max(time) FROM logs")][0][0]
-
+ should_be = numlogs
+ starttime = int(time.time())
+ endtime = 0
+ for doc in self.livestatus_broker.db.conn.bigbigbig.logs.find():
+ if doc['time'] < starttime:
+ starttime = doc['time']
+ if doc['time'] > endtime:
+ endtime = doc['time']
+ print "starttime, endtime", starttime, endtime
# now we have a lot of events
# find type = HOST ALERT for test_host_005
@@ -452,6 +465,7 @@ Or: 8
Filter: host_name = test_host_099
Filter: service_description = test_ok_01
And: 5
+Filter: plugin_output ~ HUHU
OutputFormat: json"""
# switch back to realtime. we want to know how long it takes
fake_time_time = time.time
@@ -461,13 +475,20 @@ OutputFormat: json"""
print request
response, keepalive = self.livestatus_broker.livestatus.handle_request(request)
pyresponse = eval(response)
- print "sent records", num_log_broks
- print "number of records", len(pyresponse)
+ print "number of all documents", self.livestatus_broker.db.conn.bigbigbig.logs.find().count()
+ print "number of log broks sent", num_log_broks
+ print "number of lines in the response", len(pyresponse)
print "should be", should_be
time.time = fake_time_time
time.sleep = fake_time_sleep
-
-
+ hosts = set([h[4] for h in pyresponse])
+ services = set([h[5] for h in pyresponse])
+ print "found hosts", hosts
+ print "found services", services
+ alldocs = [d for d in self.livestatus_broker.db.conn.bigbigbig.logs.find()]
+ clientselected = [d for d in alldocs if (d['time'] >= int(starttime) and d['time'] <= int(endtime) and d['host_name'] == 'test_host_099' and d['service_description'] == 'test_ok_01' and 'HUHU' in d['plugin_output'])]
+ print "clientselected", len(clientselected)
+ self.assert_(len(pyresponse) == len(clientselected))
if __name__ == '__main__':
--
UNNAMED PROJECT
More information about the Pkg-nagios-changes
mailing list