[Pkg-nagios-changes] [SCM] UNNAMED PROJECT branch, debian/master, updated. 810edbdd3feedbfe37f4a65bee50b57b2f60fa2a

Gerhard Lausser gerhard.lausser at consol.de
Tue Feb 28 22:15:41 UTC 2012


The following commit has been merged in the debian/master branch:
commit 41a218e4aafa1f17a4b5ae93b2a540298c902390
Author: Gerhard Lausser <gerhard.lausser at consol.de>
Date:   Sun Jan 22 21:27:40 2012 +0100

    More bugfixes in the MongoDB log storage and a better test

diff --git a/shinken/modules/logstore_mongodb.py b/shinken/modules/logstore_mongodb.py
index f24f46b..66c4f08 100644
--- a/shinken/modules/logstore_mongodb.py
+++ b/shinken/modules/logstore_mongodb.py
@@ -53,6 +53,7 @@ class LiveStatusLogStoreMongoDB(BaseModule):
         # mongodb://host1,host2,host3/?safe=true;w=2;wtimeoutMS=2000
         self.mongodb_uri = getattr(modconf, 'mongodb_uri', None)
         self.database = getattr(modconf, 'database', 'logs')
+        self.collection = getattr(modconf, 'collection', 'logs')
         self.use_aggressive_sql = True
         max_logs_age = getattr(modconf, 'max_logs_age', '365')
         maxmatch = re.match(r'^(\d+)([dwm]*)$', max_logs_age)
@@ -92,7 +93,7 @@ class LiveStatusLogStoreMongoDB(BaseModule):
         try:
             self.conn = pymongo.Connection(self.mongodb_uri, fsync=True)
             self.db = self.conn[self.database]
-            
+            self.db[self.collection].ensure_index([('time', pymongo.ASCENDING), ('lineno', pymongo.ASCENDING)], name='time_idx')
             self.is_connected = True
         except AutoReconnect, exp:
             # now what, ha?
@@ -122,7 +123,10 @@ class LiveStatusLogStoreMongoDB(BaseModule):
             print "Unexpected error:", exp
         try:
             if logline.logclass != LOGCLASS_INVALID:
-                self.db.logs.insert(values, safe=True)
+                self.db[self.collection].insert(values, safe=True)
+            else:
+                print "This line is invalid", line
+
         except Exception, exp:
             print "An error occurred:", exp
             print "DATABASE ERROR!!!!!!!!!!!!!!!!!"
@@ -162,15 +166,15 @@ class LiveStatusLogStoreMongoDB(BaseModule):
         # We can apply the filterstack here as well. we have columns and filtercolumns.
         # the only additional step is to enrich log lines with host/service-attributes
         # A timerange can be useful for a faster preselection of lines
-        filter_element = eval(mongo_filter)
-        print "mongo filter iis", type(filter_element)
-        print "mongo filter iis", filter_element
+        filter_element = eval('{ '+mongo_filter+' }')
+        print "mongo filter is", filter_element
         dbresult = []
         columns = ['logobject', 'attempt', 'logclass', 'command_name', 'comment', 'contact_name', 'host_name', 'lineno', 'message', 'options', 'plugin_output', 'service_description', 'state', 'state_type', 'time', 'type']
         if not self.is_connected:
             print "sorry, not connected"
         else:
-            dbresult = [Logline([(c, ) for c in columns], [x[col] for col in columns]) for x in self.db.logs.find(filter_element)]
+            dbresult = [Logline([(c, ) for c in columns], [x[col] for col in columns]) for x in self.db[self.collection].find(filter_element).sort([(u'time', pymongo.ASCENDING), (u'lineno', pymongo.ASCENDING)])]
+        print "i count", self.db[self.collection].find(filter_element).count()
         return dbresult
 
     def make_mongo_filter(self, operator, attribute, reference):
@@ -180,8 +184,9 @@ class LiveStatusLogStoreMongoDB(BaseModule):
         good_attributes = ['time', 'attempt', 'class', 'command_name', 'comment', 'contact_name', 'host_name', 'plugin_output', 'service_description', 'state', 'state_type', 'type']
         good_operators = ['=', '!=']
         #  put strings in '' for the query
-        if attribute in ['command_name', 'comment', 'contact_name', 'host_name', 'plugin_output', 'service_description', 'state_type', 'type']:
-            attribute = "'%s'" % attribute
+        string_attributes = ['command_name', 'comment', 'contact_name', 'host_name', 'plugin_output', 'service_description', 'state_type', 'type']
+        if attribute in string_attributes:
+            reference = "'%s'" % reference
 
         def eq_filter():
             if reference == '':
@@ -202,9 +207,9 @@ class LiveStatusLogStoreMongoDB(BaseModule):
         def le_filter():
             return '\'%s\' : { \'$lte\' : %s }' % (attribute, reference)
         def match_filter():
-            return '\'%s\' : { \'$regex\' : \'%s\' }' % (attribute, reference)
+            return '\'%s\' : { \'$regex\' : %s }' % (attribute, reference)
         def no_filter():
-            return '{}'
+            return '\'%s\' : { \'$exists\' : true }' % (attribute,)
         if attribute not in good_attributes:
             return no_filter
         if operator == '=':
@@ -256,7 +261,7 @@ class LiveStatusMongoStack(LiveStatusStack):
             # Make a combined anded function
             # Put it on the stack
             print "filter is", filters
-            and_clause = lambda: '{\'$and\' : [%s]}' % ', '.join('{ ' + x() + ' }' for x in filters)
+            and_clause = lambda: '\'$and\' : [%s]' % ', '.join('{ ' + x() + ' }' for x in filters)
             print "and_elements", and_clause
             self.put_stack(and_clause)
 
@@ -266,7 +271,7 @@ class LiveStatusMongoStack(LiveStatusStack):
             filters = []
             for _ in range(num):
                 filters.append(self.get_stack())
-            or_clause = lambda: '{\'$or\' : [%s]}' % ', '.join('{ ' + x() + ' }' for x in filters)
+            or_clause = lambda: '\'$or\' : [%s]' % ', '.join('{ ' + x() + ' }' for x in filters)
             print "or_elements", or_clause
             self.put_stack(or_clause)
 
diff --git a/shinken/modules/logstore_sqlite.py b/shinken/modules/logstore_sqlite.py
index f22996a..1e13776 100644
--- a/shinken/modules/logstore_sqlite.py
+++ b/shinken/modules/logstore_sqlite.py
@@ -380,9 +380,8 @@ class LiveStatusLogStoreSqlite(BaseModule):
         #FIXME need access to this#self.livestatus.count_event('log_message')
 
     def add_filter(self, operator, attribute, reference):
-        print "isql add", operator, attribute, reference
 	if attribute == 'time':
-	    #self.sql_time_filter_stack.put_stack(self.make_sql_filter(operator, attribute, reference))
+	    self.sql_time_filter_stack.put_stack(self.make_sql_filter(operator, attribute, reference))
             pass
 	self.sql_filter_stack.put_stack(self.make_sql_filter(operator, attribute, reference))
 
@@ -526,7 +525,6 @@ class LiveStatusSqlStack(LiveStatusStack):
             filters = []
             for _ in range(num):
                 filters.append(self.get_stack())
-            print "now i and", [x() for x in filters]
             # Take from the stack:
             # Make a combined anded function
             # Put it on the stack
diff --git a/test/test_livestatus_db.py b/test/test_livestatus_db.py
index 5150a42..b79c3db 100755
--- a/test/test_livestatus_db.py
+++ b/test/test_livestatus_db.py
@@ -623,23 +623,6 @@ OutputFormat: json"""
         pyresponse = eval(response)
         print "number of records", len(pyresponse)
         print "should be", should_be
-        numlogs = self.livestatus_broker.db.execute("SELECT min(time), max(time) FROM logs")
-        print starttime, endtime, numlogs
-        self.livestatus_broker.livestatus.use_aggressive_sql = True
-        print "aggrosql", self.livestatus_broker.livestatus.use_aggressive_sql
-        response2, keepalive = self.livestatus_broker.livestatus.handle_request(request)
-        self.assert_(response2 == response)
-        print "aggrosql", self.livestatus_broker.livestatus.use_aggressive_sql
-        response2, keepalive = self.livestatus_broker.livestatus.handle_request(request)
-        self.assert_(response2 == response)
-        self.livestatus_broker.livestatus.use_aggressive_sql = False
-        print "aggrosql", self.livestatus_broker.livestatus.use_aggressive_sql
-        response2, keepalive = self.livestatus_broker.livestatus.handle_request(request)
-        self.assert_(response2 == response)
-        print "aggrosql", self.livestatus_broker.livestatus.use_aggressive_sql
-        response2, keepalive = self.livestatus_broker.livestatus.handle_request(request)
-        self.assert_(response2 == response)
-        # back to fake time for the other tests can run faster
         time.time = fake_time_time
         time.sleep = fake_time_sleep
 
diff --git a/test/test_livestatus_mongodb.py b/test/test_livestatus_mongodb.py
index 0ec2400..5b1da9c 100755
--- a/test/test_livestatus_mongodb.py
+++ b/test/test_livestatus_mongodb.py
@@ -279,8 +279,8 @@ class TestConfigBig(TestConfig):
         host.__class__.use_aggressive_host_checking = 1
 
 
-    def xinit_livestatus(self):
-        self.livelogs = 'tmp/livelogs.db' + "wrumm"
+    def init_livestatus(self):
+        self.livelogs = "bigbigbig"
         modconf = Module({'module_name' : 'LiveStatus',
             'module_type' : 'livestatus',
             'port' : str(50000 + os.getpid()),
@@ -292,6 +292,7 @@ class TestConfigBig(TestConfig):
 
         dbmodconf = Module({'module_name' : 'LogStore',
             'module_type' : 'logstore_mongodb',
+            'database' : 'bigbigbig',
             'mongodb_uri' : "mongodb://127.0.0.1:27017",
         })
         modconf.modules = [dbmodconf]
@@ -346,7 +347,11 @@ class TestConfigBig(TestConfig):
         starttime = time.time()
 
         num_log_broks = 0
-        if True:
+        try:
+            numlogs = self.livestatus_broker.db.conn.bigbigbig.find().count()
+        except Exception:
+            numlogs = 0
+        if numlogs == 0:
             # run silently
             old_stdout = sys.stdout
             sys.stdout = open(os.devnull, "w")
@@ -360,6 +365,8 @@ class TestConfigBig(TestConfig):
             num_log_broks += self.count_log_broks()
             self.update_broker()
             should_be = 0
+            should_be_huhu = 0
+            huhuhus = []
             #for i in xrange(3600 * 24 * 7):
             for i in xrange(10000): 
                 if i % 1000 == 0:
@@ -370,9 +377,11 @@ class TestConfigBig(TestConfig):
                         [test_ok_01, 2, "CRIT"],
                         [test_ok_04, 3, "UNKN"],
                         [test_ok_16, 1, "WARN"],
-                        [test_ok_99, 2, "CRIT"],
+                        [test_ok_99, 2, "HUHU"+str(i)],
                     ])
                     should_be += 3
+                    should_be_huhu += 3
+                    huhuhus.append(i)
                 time.sleep(62)
                 if i % 399 == 0:
                     self.scheduler_loop(1, [
@@ -423,11 +432,15 @@ class TestConfigBig(TestConfig):
             sys.stdout = old_stdout
             self.livestatus_broker.db.commit()
         else:
-            should_be = numlogs[0][0]
-            xxx = self.livestatus_broker.db.execute("SELECT min(time), max(time) FROM logs")
-            print xxx
-            starttime, endtime = [self.livestatus_broker.db.execute("SELECT min(time), max(time) FROM logs")][0][0]
-            
+            should_be = numlogs
+            starttime = int(time.time())
+            endtime = 0
+            for doc in self.livestatus_broker.db.conn.bigbigbig.logs.find():
+                if doc['time'] < starttime:
+                    starttime = doc['time']
+                if doc['time'] > endtime:
+                    endtime = doc['time']
+            print "starttime, endtime", starttime, endtime
         
         # now we have a lot of events
         # find type = HOST ALERT for test_host_005
@@ -452,6 +465,7 @@ Or: 8
 Filter: host_name = test_host_099
 Filter: service_description = test_ok_01
 And: 5
+Filter: plugin_output ~ HUHU
 OutputFormat: json"""
         # switch back to realtime. we want to know how long it takes
         fake_time_time = time.time
@@ -461,13 +475,20 @@ OutputFormat: json"""
         print request
         response, keepalive = self.livestatus_broker.livestatus.handle_request(request)
         pyresponse = eval(response)
-        print "sent records", num_log_broks
-        print "number of records", len(pyresponse)
+        print "number of all documents", self.livestatus_broker.db.conn.bigbigbig.logs.find().count()
+        print "number of log broks sent", num_log_broks
+        print "number of lines in the response", len(pyresponse)
         print "should be", should_be
         time.time = fake_time_time
         time.sleep = fake_time_sleep
-
-
+        hosts = set([h[4] for h in pyresponse])
+        services = set([h[5] for h in pyresponse])
+        print "found hosts", hosts
+        print "found services", services
+        alldocs = [d for d in self.livestatus_broker.db.conn.bigbigbig.logs.find()]
+        clientselected = [d for d in alldocs if (d['time'] >= int(starttime) and d['time'] <= int(endtime) and d['host_name'] == 'test_host_099' and d['service_description'] == 'test_ok_01' and 'HUHU' in d['plugin_output'])]
+        print "clientselected", len(clientselected)
+        self.assert_(len(pyresponse) == len(clientselected))
 
 
 if __name__ == '__main__':

-- 
UNNAMED PROJECT



More information about the Pkg-nagios-changes mailing list