[med-svn] [Git][med-team/gnumed-server][upstream] New upstream version 22.13
Andreas Tille
gitlab at salsa.debian.org
Fri Jul 24 08:17:21 BST 2020
Andreas Tille pushed to branch upstream at Debian Med / gnumed-server
Commits:
edf8d011 by Andreas Tille at 2020-07-24T09:03:39+02:00
New upstream version 22.13
- - - - -
18 changed files:
- server/bootstrap/bootstrap_gm_db_system.py
- server/doc/schema/gnumed-entire_schema.html
- server/gm-fingerprint_db.py
- server/pycommon/gmBackendListener.py
- server/pycommon/gmBusinessDBObject.py
- server/pycommon/gmCfg2.py
- server/pycommon/gmConnectionPool.py
- server/pycommon/gmDateTime.py
- server/pycommon/gmDispatcher.py
- server/pycommon/gmGuiBroker.py
- server/pycommon/gmI18N.py
- server/pycommon/gmLog2.py
- server/pycommon/gmLoginInfo.py
- server/pycommon/gmMimeLib.py
- server/pycommon/gmNetworkTools.py
- server/pycommon/gmPG2.py
- server/pycommon/gmTools.py
- server/sql/v21-v22/fixups/v22-release_notes-fixup.sql
Changes:
=====================================
server/bootstrap/bootstrap_gm_db_system.py
=====================================
@@ -290,7 +290,7 @@ def connect(host, port, db, user, passwd, conn_name=None):
#==================================================================
class user:
- def __init__(self, anAlias = None, aPassword = None):
+ def __init__(self, anAlias = None, aPassword = None, force_interactive=False):
if anAlias is None:
raise ConstructorError("need user alias")
self.alias = anAlias
@@ -314,10 +314,21 @@ class user:
# defined but empty:
# this means to ask the user if interactive
elif self.password == '':
- if _interactive:
+ if _interactive or force_interactive:
+ _log.info('password for [%s] defined as "", asking user', self.name)
print("I need the password for the database user [%s]." % self.name)
self.password = getpass.getpass("Please type the password: ")
+ _log.info('got password')
+ pwd4check = None
+ while pwd4check != self.password:
+ _log.info('asking for confirmation')
+ pwd2 = getpass.getpass("Please retype the password: ")
+ if pwd2 == self.password:
+ break
+ _log.error('password mismatch, asking again')
+ print('Password mismatch. Try again or CTRL-C to abort.')
else:
+ _log.warning('password for [%s] defined as "" (meaning <ask-user>), but running non-interactively, aborting', self.name)
_log.warning('cannot get password for database user [%s]', self.name)
raise ValueError('no password for user %s' % self.name)
@@ -495,7 +506,7 @@ unless it is pre-defined in the configuration file.
Make sure to remember the password for later use !
""") % name)
- _dbowner = user(anAlias = dbowner_alias)
+ _dbowner = user(anAlias = dbowner_alias, force_interactive = True)
cmd = 'create user "%s" with password \'%s\' createdb createrole in group "%s", "gm-logins"' % (_dbowner.name, _dbowner.password, self.auth_group)
try:
@@ -615,6 +626,18 @@ class database:
_log.error(u"Cannot connect to template database.")
return False
+ # fingerprint template db
+ try:
+ gmLog2.log_multiline (
+ logging.INFO,
+ message = 'template database fingerprint',
+ text = gmPG2.get_db_fingerprint(conn = self.conn, eol = '\n')
+ )
+ except Exception:
+ _log.exception('cannot fingerprint template database')
+
+ self.conn.rollback()
+
# make sure db exists
if not self.__create_db():
_log.error(u"Cannot create database.")
@@ -1754,6 +1777,24 @@ def handle_cfg():
if not import_data():
exit_with_msg("Bootstrapping failed: unable to import data")
+ # fingerprint target db
+ # (the ugliest of hacks)
+ db = None
+ for db_key in _bootstrapped_dbs.keys():
+ db = _bootstrapped_dbs[db_key]
+ if db is not None:
+ try:
+ gmLog2.log_multiline (
+ logging.INFO,
+ message = 'target database fingerprint',
+ text = gmPG2.get_db_fingerprint(conn = db.conn, eol = '\n')
+ )
+ except Exception:
+ _log.error('cannot fingerprint database')
+ finally:
+ db.conn.rollback()
+ return True
+
#==================================================================
def main():
=====================================
server/doc/schema/gnumed-entire_schema.html
=====================================
@@ -112,7 +112,7 @@
<body>
<!-- Primary Index -->
- <p><br><br>Dumped on 2020-03-25</p>
+ <p><br><br>Dumped on 2020-07-23</p>
<h1><a name="index">Index of database - gnumed_v22</a></h1>
<ul>
=====================================
server/gm-fingerprint_db.py
=====================================
@@ -12,7 +12,9 @@
import sys
import psycopg2
-import io
+#import io
+
+from Gnumed.pycommon import gmPG2
database = sys.argv[1]
passwd = sys.argv[2]
@@ -40,28 +42,33 @@ queries = [
fname = u'gm_db-%s-fingerprint.log' % database
#==============================================================
-outfile = io.open(fname, mode = 'wt', encoding = 'utf8')
+#outfile = io.open(fname, mode = 'wt', encoding = 'utf8')
-outfile.write(u"Fingerprinting GNUmed database ...\n")
-outfile.write(u"\n")
-outfile.write(u"%20s: %s\n" % (u"Name (DB)", database))
+#outfile.write(u"Fingerprinting GNUmed database ...\n")
+#outfile.write(u"\n")
+#outfile.write(u"%20s: %s\n" % (u"Name (DB)", database))
conn = psycopg2.connect(dsn=dsn)
-curs = conn.cursor()
-
-for cmd, label in queries:
- curs.execute(cmd)
- rows = curs.fetchall()
- outfile.write(u"%20s: %s\n" % (label, rows[0][0]))
-
+#curs = conn.cursor()
+#
+#for cmd, label in queries:
+# curs.execute(cmd)
+# rows = curs.fetchall()
+# outfile.write(u"%20s: %s\n" % (label, rows[0][0]))
+#
if len(sys.argv) > 3:
if sys.argv[3] == '--with-dump':
- curs.execute('SELECT gm.concat_table_structure()')
- rows = curs.fetchall()
- outfile.write(u"\n%s\n" % rows[0][0])
+ with_dump = True
+ else:
+ with_dump = False
+# curs.execute('SELECT gm.concat_table_structure()')
+# rows = curs.fetchall()
+# outfile.write(u"\n%s\n" % rows[0][0])
+
+gmPG2.fingerprint_db(conn = conn, with_dump = with_dump, fname = fname)
-curs.close()
+#curs.close()
conn.rollback()
-outfile.close()
+#outfile.close()
#==============================================================
=====================================
server/pycommon/gmBackendListener.py
=====================================
@@ -393,7 +393,7 @@ if __name__ == "__main__":
except KeyError:
print('==> received signal from client: "%s"' % kwargs['signal'])
del kwargs['signal']
- for key in kwargs.keys():
+ for key in kwargs:
print(' [%s]: %s' % (key, kwargs[key]))
gmDispatcher.connect(receiver = monitoring_callback)
=====================================
server/pycommon/gmBusinessDBObject.py
=====================================
@@ -355,7 +355,7 @@ def manage_xxx()
result = self.refetch_payload(link_obj = link_obj)
if result is True:
self.payload_most_recently_fetched = {}
- for field in self._idx.keys():
+ for field in self._idx:
self.payload_most_recently_fetched[field] = self._payload[self._idx[field]]
return True
@@ -398,7 +398,7 @@ def manage_xxx()
self.pk_obj = row['pk_obj']
self.payload_most_recently_fetched = {}
- for field in self._idx.keys():
+ for field in self._idx:
self.payload_most_recently_fetched[field] = self._payload[self._idx[field]]
#--------------------------------------------------------
@@ -413,7 +413,7 @@ def manage_xxx()
def __str__(self):
lines = []
try:
- for attr in self._idx.keys():
+ for attr in self._idx:
if self._payload[self._idx[attr]] is None:
lines.append('%s: NULL' % attr)
else:
@@ -440,7 +440,7 @@ def manage_xxx()
getter = getattr(self, 'get_%s' % attribute, None)
if not callable(getter):
_log.warning('[%s]: no attribute [%s]' % (self.__class__.__name__, attribute))
- _log.warning('[%s]: valid attributes: %s' % (self.__class__.__name__, str(self._idx.keys())))
+ _log.warning('[%s]: valid attributes: %s', self.__class__.__name__, list(self._idx))
_log.warning('[%s]: no getter method [get_%s]' % (self.__class__.__name__, attribute))
methods = [ m for m in inspect.getmembers(self, inspect.ismethod) if m[0].startswith('get_') ]
_log.warning('[%s]: valid getter methods: %s' % (self.__class__.__name__, str(methods)))
@@ -502,7 +502,7 @@ def manage_xxx()
#--------------------------------------------------------
def get_fields(self):
try:
- return self._idx.keys()
+ return list(self._idx)
except AttributeError:
return 'nascent [%s @ %s], cannot return keys' %(self.__class__.__name__, id(self))
@@ -517,7 +517,7 @@ def manage_xxx()
else:
bools = {True: bool_strings[0], False: bool_strings[1]}
data = {}
- for field in self._idx.keys():
+ for field in self._idx:
# FIXME: harden against BYTEA fields
#if type(self._payload[self._idx[field]]) == ...
# data[field] = _('<%s bytes of binary data>') % len(self._payload[self._idx[field]])
@@ -707,7 +707,7 @@ def manage_xxx()
return (True, None)
args = {}
- for field in self._idx.keys():
+ for field in self._idx:
args[field] = self._payload[self._idx[field]]
self.payload_most_recently_attempted_to_store = args
@@ -763,7 +763,7 @@ def manage_xxx()
# update to new "original" payload
self.payload_most_recently_fetched = {}
- for field in self._idx.keys():
+ for field in self._idx:
self.payload_most_recently_fetched[field] = self._payload[self._idx[field]]
return (True, None)
=====================================
server/pycommon/gmCfg2.py
=====================================
@@ -388,7 +388,7 @@ class gmCfgData(gmBorg.cBorg):
try: source_data = self.__cfg_data[source]
except KeyError:
_log.error('invalid config source [%s]', source)
- _log.debug('currently known sources: %s', self.__cfg_data.keys())
+ _log.debug('currently known sources: %s', list(self.__cfg_data))
continue
try: value = source_data[option_path]
=====================================
server/pycommon/gmConnectionPool.py
=====================================
@@ -12,6 +12,9 @@ __author__ = "karsten.hilbert at gmx.net"
__license__ = "GPL v2 or later (details at http://www.gnu.org)"
+_DISABLE_CONNECTION_POOL = False # set to True to disable the connection pool for debugging (= always return new connection)
+
+
# standard library imports
import os
import sys
@@ -29,7 +32,7 @@ import psycopg2 as dbapi
if not (float(dbapi.apilevel) >= 2.0):
raise ImportError('gmPG2: supported DB-API level too low')
-if not (dbapi.threadsafety > 0):
+if not (dbapi.threadsafety == 2):
raise ImportError('gmPG2: lacking minimum thread safety in psycopg2')
if not (dbapi.paramstyle == 'pyformat'):
@@ -50,6 +53,7 @@ try:
except ValueError:
raise ImportError('gmPG2: lacking v3 backend protocol support in psycopg2')
+
import psycopg2.extensions
import psycopg2.extras
import psycopg2.errorcodes as SQL_error_codes
@@ -84,6 +88,10 @@ _log.info('psycopg2 module version: %s' % dbapi.__version__)
_log.info('PostgreSQL via DB-API module "%s": API level %s, thread safety %s, parameter style "%s"' % (dbapi, dbapi.apilevel, dbapi.threadsafety, dbapi.paramstyle))
_log.info('libpq version (compiled in): %s', psycopg2.__libpq_version__)
_log.info('libpq version (loaded now) : %s', psycopg2.extensions.libpq_version())
+#if '2.8' in dbapi.__version__:
+# _log.info('psycopg2 v2.8 detected, disabling connection pooling for the time being')
+# _DISABLE_CONNECTION_POOL = True
+
postgresql_version = None
@@ -117,54 +125,52 @@ _map_psyco_iso_level2str = {
class cPGCredentials:
def __init__(self) -> None:
- self.__host = None # None: left out of DSN -> defaults to $PGHOST or implicit <localhost>
- self.__port = None # None: left out of DSN -> defaults to $PGPORT or libpq compiled-in default (typically 5432)
+ self.__host = None # None: left out -> defaults to $PGHOST or implicit <localhost>
+ self.__port = None # None: left out -> defaults to $PGPORT or libpq compiled-in default (typically 5432)
self.__database = None # must be set before connecting
self.__user = None # must be set before connecting
- self.__password = None # None: left out of DSN
+ self.__password = None # None: left out
# -> try password-less connect (TRUST/IDENT/PEER)
- # -> try connect with password from DSN:<passfile> or $PGPASSFILE or ~/.pgpass
+ # -> try connect with password from <passfile> parameter or $PGPASSFILE or ~/.pgpass
#--------------------------------------------------
# properties
#--------------------------------------------------
def __format_credentials(self):
- dsn_parts = [
+ cred_parts = [
'dbname=%s' % self.__database,
'host=%s' % self.__host,
'port=%s' % self.__port,
'user=%s' % self.__user
]
- return ' '.join(dsn_parts)
+ return ' '.join(cred_parts)
- formatted_dsn = property(__format_credentials)
+ formatted_credentials = property(__format_credentials)
#--------------------------------------------------
- def generate_dsn(self, connection_name=None):
+ def generate_credentials_kwargs(self, connection_name=None):
assert (self.__database is not None), 'self.__database must be defined'
assert (self.__user is not None), 'self.__user must be defined'
-
- dsn_parts = [
- 'dbname=%s' % self.__database,
- #'port=%s' % gmTools.coalesce(self.__port, '5432'),
- 'user=%s' % self.__user,
- 'application_name=%s' % gmTools.coalesce(connection_name, 'GNUmed'),
- 'fallback_application_name=GNUmed',
- 'sslmode=prefer',
+ kwargs = {
+ 'dbname': self.__database,
+ 'user': self.__user,
+ 'application_name': gmTools.coalesce(connection_name, 'GNUmed'),
+ 'fallback_application_name': 'GNUmed',
+ 'sslmode': 'prefer',
# try to enforce a useful encoding early on so that we
# have a good chance of decoding authentication errors
# containing foreign language characters
- 'client_encoding=UTF8'
- ]
+ 'client_encoding': 'UTF8'
+ }
if self.__host is not None:
- dsn_parts.append('host=%s' % self.__host)
+ kwargs['host'] = self.__host
if self.__port is not None:
- dsn_parts.append('port=%s' % self.__port)
+ kwargs['port'] = self.__port
if self.__password is not None:
- dsn_parts.append('password=%s' % self.__password)
- return ' '.join(dsn_parts)
+ kwargs['password'] = self.__password
+ return kwargs
- validated_dsn = property(generate_dsn)
+ credentials_kwargs = property(generate_credentials_kwargs)
#--------------------------------------------------
def _get_database(self):
@@ -240,7 +246,7 @@ class gmConnectionPool(gmBorg.cBorg):
self.__initialized = True
_log.info('[%s]: first instantiation', self.__class__.__name__)
- self.__ro_conn_pool = {} # keyed by "DSN::thread ID"
+ self.__ro_conn_pool = {} # keyed by "credentials::thread ID"
self.__SQL_set_client_timezone = None
self.__client_timezone = None
self.__creds = None
@@ -250,6 +256,10 @@ class gmConnectionPool(gmBorg.cBorg):
# connection API
#--------------------------------------------------
def get_connection(self, readonly=True, verbose=False, pooled=True, connection_name=None, autocommit=False, credentials=None):
+
+ if _DISABLE_CONNECTION_POOL:
+ pooled = False
+
if credentials is not None:
pooled = False
conn = None
@@ -328,12 +338,12 @@ class gmConnectionPool(gmBorg.cBorg):
creds2use = self.__creds
else:
creds2use = credentials
- dsn = creds2use.generate_dsn(connection_name = connection_name)
+ creds_kwargs = creds2use.generate_credentials_kwargs(connection_name = connection_name)
try:
# DictConnection now _is_ a real dictionary
- conn = dbapi.connect(dsn = dsn, connection_factory = psycopg2.extras.DictConnection)
+ conn = dbapi.connect(connection_factory = psycopg2.extras.DictConnection, **creds_kwargs)
except dbapi.OperationalError as e:
- _log.error('failed to establish connection [%s]', creds2use.formatted_dsn)
+ _log.error('failed to establish connection [%s]', creds2use.formatted_credentials)
t, v, tb = sys.exc_info()
try:
msg = e.args[0]
@@ -343,7 +353,7 @@ class gmConnectionPool(gmBorg.cBorg):
if not self.__is_auth_fail_msg(msg):
raise
- raise cAuthenticationError(creds2use.formatted_dsn, msg).with_traceback(tb)
+ raise cAuthenticationError(creds2use.formatted_credentials, msg).with_traceback(tb)
_log.debug('established connection "%s", backend PID: %s', gmTools.coalesce(connection_name, 'anonymous'), conn.get_backend_pid())
# - inspect server
@@ -580,9 +590,9 @@ class gmConnectionPool(gmBorg.cBorg):
def _set_credentials(self, creds=None):
if self.__creds is not None:
_log.debug('invalidating pooled connections on credentials change')
- curr_dsn = self.__creds.formatted_dsn + '::'
+ curr_creds = self.__creds.formatted_credentials + '::'
for conn_key in self.__ro_conn_pool:
- if not conn_key.startswith(curr_dsn):
+ if not conn_key.startswith(curr_creds):
continue
conn = self.__ro_conn_pool[conn_key]
self.__ro_conn_pool[conn_key] = None
@@ -601,7 +611,7 @@ class gmConnectionPool(gmBorg.cBorg):
#--------------------------------------------------
def _get_pool_key(self):
return '%s::%s' % (
- self.__creds.formatted_dsn,
+ self.__creds.formatted_credentials,
threading.current_thread().ident
)
@@ -868,12 +878,12 @@ def _raise_exception_on_pooled_ro_conn_close():
#========================================================================
class cAuthenticationError(dbapi.OperationalError):
- def __init__(self, dsn=None, prev_val=None):
- self.dsn = dsn
+ def __init__(self, creds=None, prev_val=None):
+ self.creds = creds
self.prev_val = prev_val
def __str__(self):
- return 'PostgreSQL: %sDSN: %s' % (self.prev_val, self.dsn)
+ return 'PostgreSQL: %sDSN: %s' % (self.prev_val, self.creds)
#============================================================
# Python -> PostgreSQL
@@ -920,7 +930,7 @@ if __name__ == "__main__":
print("testing exceptions")
try:
- raise cAuthenticationError('no dsn', 'no previous exception')
+ raise cAuthenticationError('no credentials', 'no previous exception')
except cAuthenticationError:
t, v, tb = sys.exc_info()
print(t)
@@ -1047,6 +1057,22 @@ if __name__ == "__main__":
conn = pool.get_connection()
#--------------------------------------------------------------------
- test_exceptions()
- test_get_connection()
- test_change_creds()
+ def test_credentials():
+ print("testing credentials with spaces")
+ pool = gmConnectionPool()
+ creds = cPGCredentials()
+ creds.database = 'gnumed_v22'
+ creds.user = 'any-doc'
+ creds.password = 'any-doc'
+ pool.credentials = creds
+ conn = pool.get_connection()
+ print(conn)
+ creds.password = 'a - b'
+ pool.credentials = creds
+ conn = pool.get_connection()
+
+ #--------------------------------------------------------------------
+ test_credentials()
+ #test_exceptions()
+ #test_get_connection()
+ #test_change_creds()
=====================================
server/pycommon/gmDateTime.py
=====================================
@@ -2162,7 +2162,7 @@ if __name__ == '__main__':
#-----------------------------------------------------------------------
def test_format_interval():
intv = pyDT.timedelta(minutes=1, seconds=2)
- for acc in _accuracy_strings.keys():
+ for acc in _accuracy_strings:
print ('[%s]: "%s" -> "%s"' % (acc, intv, format_interval(intv, acc)))
return
@@ -2171,7 +2171,7 @@ if __name__ == '__main__':
if intv is None:
print(tmp, '->', intv)
continue
- for acc in _accuracy_strings.keys():
+ for acc in _accuracy_strings:
print ('[%s]: "%s" -> "%s"' % (acc, tmp, format_interval(intv, acc)))
#-----------------------------------------------------------------------
=====================================
server/pycommon/gmDispatcher.py
=====================================
@@ -322,7 +322,7 @@ def _call(receiver, **kwds):
if not (func_code_def.co_flags & 0x08):
# func_code_def does not have a **kwds type parameter,
# therefore remove unacceptable arguments.
- keys = list(kwds.keys())
+ keys = list(kwds)
for arg in keys:
if arg not in acceptable_args:
del kwds[arg]
@@ -337,8 +337,8 @@ def _call(receiver, **kwds):
#---------------------------------------------------------------------
def _removeReceiver(receiver):
"""Remove receiver from connections."""
- for sender_identity in connections.keys():
- for signal in connections[sender_identity].keys():
+ for sender_identity in connections:
+ for signal in connections[sender_identity]:
receivers = connections[sender_identity][signal]
try:
receivers.remove(receiver)
=====================================
server/pycommon/gmGuiBroker.py
=====================================
@@ -70,7 +70,7 @@ class GuiBroker:
def keylist(self):
" returns a list of all keys; see documentation for the dictionary data type"
- return GuiBroker.__objects.keys()
+ return list(GuiBroker.__objects)
=====================================
server/pycommon/gmI18N.py
=====================================
@@ -107,37 +107,33 @@ def __split_locale_into_levels():
#---------------------------------------------------------------------------
def __log_locale_settings(message=None):
_setlocale_categories = {}
- for category in 'LC_ALL LC_CTYPE LC_COLLATE LC_TIME LC_MONETARY LC_MESSAGES LC_NUMERIC'.split():
+ for category in 'LC_ALL LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT LC_IDENTIFICATION'.split():
try:
_setlocale_categories[category] = getattr(locale, category)
except Exception:
_log.warning('this OS does not have locale.%s', category)
-
_getlocale_categories = {}
- for category in 'LC_CTYPE LC_COLLATE LC_TIME LC_MONETARY LC_MESSAGES LC_NUMERIC'.split():
+ for category in 'LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT LC_IDENTIFICATION'.split():
try:
_getlocale_categories[category] = getattr(locale, category)
except Exception:
pass
-
if message is not None:
_log.debug(message)
_log.debug('current locale settings:')
_log.debug('locale.getlocale(): %s' % str(locale.getlocale()))
- for category in _getlocale_categories.keys():
+ for category in _getlocale_categories:
_log.debug('locale.getlocale(%s): %s' % (category, locale.getlocale(_getlocale_categories[category])))
-
- for category in _setlocale_categories.keys():
+ for category in _setlocale_categories:
_log.debug('(locale.setlocale(%s): %s)' % (category, locale.setlocale(_setlocale_categories[category])))
-
try:
_log.debug('locale.getdefaultlocale() - default (user) locale: %s' % str(locale.getdefaultlocale()))
except ValueError:
_log.exception('the OS locale setup seems faulty')
_log.debug('encoding sanity check (also check "locale.nl_langinfo(CODESET)" below):')
- pref_loc_enc = locale.getpreferredencoding(do_setlocale=False)
+ pref_loc_enc = locale.getpreferredencoding(do_setlocale = False)
loc_enc = locale.getlocale()[1]
py_str_enc = sys.getdefaultencoding()
sys_fs_enc = sys.getfilesystemencoding()
@@ -171,7 +167,7 @@ def __log_locale_settings(message=None):
_log.debug('database of locale conventions:')
data = locale.localeconv()
- for key in data.keys():
+ for key in data:
if loc_enc is None:
_log.debug('locale.localeconv(%s): %s', key, data[key])
else:
@@ -186,7 +182,7 @@ def __log_locale_settings(message=None):
except Exception:
_log.warning('this OS does not support nl_langinfo category locale.%s' % category)
try:
- for category in _nl_langinfo_categories.keys():
+ for category in _nl_langinfo_categories:
if loc_enc is None:
_log.debug('locale.nl_langinfo(%s): %s' % (category, locale.nl_langinfo(_nl_langinfo_categories[category])))
else:
@@ -231,37 +227,25 @@ def _translate_safely(term):
def activate_locale():
"""Get system locale from environment."""
global system_locale
-
- __log_locale_settings('unmodified startup locale settings (should be [C])')
-
- # activate user-preferred locale
+ __log_locale_settings('unmodified startup locale settings (could be [C])')
loc, enc = None, None
+ # activate user-preferred locale
try:
- # check whether already set
- loc, loc_enc = locale.getlocale()
- if loc is None:
- loc = locale.setlocale(locale.LC_ALL, '')
- _log.debug("activating user-default locale with <locale.setlocale(locale.LC_ALL, '')> returns: [%s]" % loc)
- else:
- _log.info('user-default locale already activated')
- loc, loc_enc = locale.getlocale()
+ loc = locale.setlocale(locale.LC_ALL, '')
+ _log.debug("activating user-default locale with <locale.setlocale(locale.LC_ALL, '')> returns: [%s]" % loc)
except AttributeError:
_log.exception('Windows does not support locale.LC_ALL')
except Exception:
_log.exception('error activating user-default locale')
-
__log_locale_settings('locale settings after activating user-default locale')
-
- # did we find any locale setting ? assume en_EN if not
+ # assume en_EN if we did not find any locale settings
if loc in [None, 'C']:
_log.error('the current system locale is still [None] or [C], assuming [en_EN]')
system_locale = "en_EN"
else:
system_locale = loc
-
# generate system locale levels
__split_locale_into_levels()
-
return True
#---------------------------------------------------------------------------
=====================================
server/pycommon/gmLog2.py
=====================================
@@ -339,6 +339,9 @@ if __name__ == '__main__':
#-----------------------------------------------------------
def test():
logger = logging.getLogger('gmLog2.test')
+
+ logger.error('test %s', [1,2,3])
+
logger.error("I expected to see %s::test()" % __file__)
add_word2hide('super secret passphrase')
logger.debug('credentials: super secret passphrase')
=====================================
server/pycommon/gmLoginInfo.py
=====================================
@@ -11,12 +11,6 @@ _log = logging.getLogger('gm.db')
class LoginInfo:
"""a class to encapsulate Postgres login information to default database"""
- # private variables
-# user = ''
-# password = ''
-# host = ''
-# port = 5432
-# database = ''
#------------------------------------------
def __init__(self, user=None, password=None, host=None, port=5432, database=None):
self.user = user
@@ -33,80 +27,6 @@ class LoginInfo:
port = property(_get_port, _set_port)
#------------------------------------------
- def GetInfo(self):
- return (
- self.GetUser(),
- self.GetPassword(),
- self.GetHost(),
- self.GetPort(),
- self.GetDatabase(),
- self.GetProfile()
- )
- #------------------------------------------
- def GetInfoStr(self):
- # don't hand out passwords just like that
- info = "host:port=%s:%s, db=%s, user=%s, pw=??" % (
- self.GetHost(),
- str(self.GetPort()),
- self.GetDatabase(),
- self.GetUser()
- )
- return info
- #------------------------------------------
- def GetPGDB_DSN(self):
- host = self.GetHost()
- port = str(self.GetPort())
- # for local UNIX domain sockets connections: leave host/port empty
- # IH: *PLEASE* option of local TCP/IP connection must be available
-# if host in ['', 'localhost']:
-# host = ""
- if host == '':
- port = ''
- dsn = "%s:%s:%s:%s" % (
- host,
- self.GetDatabase(),
- self.GetUser(),
- self.GetPassword()
- )
- host_port = "%s:%s" % (host, port)
- return dsn, host_port
- #------------------------------------------
- def get_psycopg2_dsn(self):
- dsn_parts = []
-
- if self.database.strip() != '':
- dsn_parts.append('dbname=%s' % self.database)
-
- if self.host.strip() != '':
- dsn_parts.append('host=%s' % self.host)
-
- dsn_parts.append('port=%s' % self.port)
-
- if self.user.strip() != '':
- dsn_parts.append('user=%s' % self.user)
-
- if self.password.strip() != '':
- dsn_parts.append('password=%s' % self.password)
-
- return ' '.join(dsn_parts)
- #------------------------------------------
- def GetDBAPI_DSN(self):
- host = self.GetHost()
- port = str(self.GetPort())
- # for local UNIX domain sockets connections: leave host/port empty
-# if host in ['', 'localhost']:
-# host = ''
- if host == '':
- port = ''
- dsn = "%s:%s:%s:%s:%s" % (
- host,
- port,
- self.GetDatabase(),
- self.GetUser(),
- self.GetPassword()
- )
- return dsn
- #------------------------------------------
def SetUser(self, user):
self.user = user
#------------------------------------------
=====================================
server/pycommon/gmMimeLib.py
=====================================
@@ -228,7 +228,7 @@ def _get_system_startfile_cmd(filename):
if _system_startfile_cmd is not None:
return True, _system_startfile_cmd % filename
- open_cmd_candidates = open_cmds.keys()
+ open_cmd_candidates = list(open_cmds)
for candidate in open_cmd_candidates:
found, binary = gmShellAPI.detect_external_binary(binary = candidate)
=====================================
server/pycommon/gmNetworkTools.py
=====================================
@@ -1,7 +1,5 @@
# -*- coding: utf-8 -*-
-
-
__doc__ = """GNUmed internetworking tools."""
#===========================================================================
@@ -155,10 +153,19 @@ def compare_versions(left_version, right_version):
-1: left < right
1: left > right
"""
+ _log.debug('comparing [%s] with [%s]', left_version, right_version)
if left_version == right_version:
- _log.debug('same version: [%s] = [%s]', left_version, right_version)
+ _log.debug('same version')
return 0
+ if right_version in ['head', 'dev', 'devel']:
+ _log.debug('development code')
+ return -1
+
+ if left_version in ['head', 'dev', 'devel']:
+ _log.debug('development code')
+ return 1
+
left_parts = left_version.split('.')
right_parts = right_version.split('.')
@@ -194,8 +201,12 @@ def check_for_update(url=None, current_branch=None, current_version=None, consid
False: up to date
None: don't know
"""
- if current_version == 'GIT HEAD':
- _log.debug('GIT HEAD always up to date')
+ if current_version is None:
+ _log.debug('<current_version> is None, currency unknown')
+ return (None, None)
+
+ if current_version.lower() in ['git head', 'head', 'tip', 'dev', 'devel']:
+ _log.debug('[%s] always considered up to date', current_version)
return (False, None)
try:
=====================================
server/pycommon/gmPG2.py
=====================================
@@ -300,6 +300,7 @@ def __request_login_params_tui():
gmLog2.add_word2hide(login.password)
login.port = prompted_input(prompt = "port", default = 5432)
except KeyboardInterrupt:
+ del login
_log.warning("user cancelled text mode login dialog")
print("user cancelled text mode login dialog")
raise gmExceptions.ConnectionError(_("Cannot connect to database without login information!"))
@@ -346,21 +347,28 @@ def __request_login_params_gui_wx():
return login, creds
#---------------------------------------------------
-def request_login_params():
+def request_login_params(setup_pool=False):
"""Request login parameters for database connection."""
-
# are we inside X ?
# if we aren't wxGTK will crash hard at the C-level with "can't open Display"
if 'DISPLAY' in os.environ:
# try wxPython GUI
try:
- return __request_login_params_gui_wx()
+ login, creds = __request_login_params_gui_wx()
except Exception:
pass
+ if setup_pool:
+ pool = gmConnectionPool.gmConnectionPool()
+ pool.credentials = creds
+ return login, creds
# well, either we are on the console or
# wxPython does not work, use text mode
- return __request_login_params_tui()
+ login, creds = __request_login_params_tui()
+ if setup_pool:
+ pool = gmConnectionPool.gmConnectionPool()
+ pool.credentials = creds
+ return login, creds
# =======================================================================
# netadata API
@@ -435,6 +443,70 @@ def get_schema_revision_history(link_obj=None):
rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd}])
return rows
+
+#------------------------------------------------------------------------
+def get_db_fingerprint(conn=None, fname=None, with_dump=False, eol=None):
+ queries = [
+ ("SELECT setting FROM pg_settings WHERE name = 'server_version'", "Version (PG)"),
+ ("SELECT setting FROM pg_settings WHERE name = 'server_encoding'", "Encoding (PG)"),
+ ("SELECT setting FROM pg_settings WHERE name = 'lc_collate'", "LC_COLLATE (PG)"),
+ ("SELECT setting FROM pg_settings WHERE name = 'lc_ctype'", "LC_CTYPE (PG)"),
+ ("SELECT count(1) FROM dem.identity", "Patients"),
+ ("SELECT count(1) FROM clin.encounter", "Contacts"),
+ ("SELECT count(1) FROM clin.episode", "Episodes"),
+ ("SELECT count(1) FROM clin.health_issue", "Issues"),
+ ("SELECT count(1) FROM clin.test_result", "Results"),
+ ("SELECT count(1) FROM clin.vaccination", "Vaccinations"),
+ ("SELECT count(1) FROM blobs.doc_med", "Documents"),
+ ("SELECT count(1) FROM blobs.doc_obj", "Objects"),
+ ("SELECT count(1) FROM dem.org", "Organizations"),
+ ("SELECT count(1) FROM dem.org_unit", "Organizational units"),
+ ("SELECT max(modified_when) FROM audit.audit_fields", "Most recent .modified_when"),
+ ("SELECT max(audit_when) FROM audit.audit_trail", "Most recent .audit_when")
+ ]
+ if conn is None:
+ conn = get_connection(readonly = True)
+ database = conn.get_dsn_parameters()['dbname']
+ lines = [
+ 'Fingerprinting GNUmed database ...',
+ '',
+ '%20s: %s' % ('Name (DB)', database)
+ ]
+ curs = conn.cursor()
+ # get size
+ cmd = "SELECT pg_size_pretty(pg_database_size('%s'))" % database
+ curs.execute(cmd)
+ rows = curs.fetchall()
+ lines.append('%20s: %s' % ('Size (DB)', rows[0][0]))
+ # get hash
+ cmd = "SELECT md5(gm.concat_table_structure())"
+ curs.execute(cmd)
+ rows = curs.fetchall()
+ md5_sum = rows[0][0]
+ try:
+ lines.append('%20s: %s (v%s)' % ('Schema hash', md5_sum, map_schema_hash2version[md5_sum]))
+ except KeyError:
+ lines.append('%20s: %s' % ('Schema hash', md5_sum))
+ for cmd, label in queries:
+ curs.execute(cmd)
+ rows = curs.fetchall()
+ lines.append('%20s: %s' % (label, rows[0][0]))
+ if with_dump:
+ curs.execute('SELECT gm.concat_table_structure()')
+ rows = curs.fetchall()
+ lines.append('')
+ lines.append(rows[0][0])
+ curs.close()
+ if fname is None:
+ if eol is None:
+ return lines
+ return eol.join(lines)
+
+ outfile = open(fname, mode = 'wt', encoding = 'utf8')
+ outfile.write('\n'.join(lines))
+ outfile.close()
+ return fname
+
#------------------------------------------------------------------------
def get_current_user():
rows, idx = run_ro_queries(queries = [{'cmd': 'select CURRENT_USER'}])
@@ -1803,7 +1875,7 @@ def run_insert(link_obj=None, schema=None, table=None, values=None, returning=No
if schema is None:
schema = 'public'
- fields = values.keys() # that way val_snippets and fields really should end up in the same order
+ fields = list(values) # that way val_snippets and fields really should end up in the same order
val_snippets = []
for field in fields:
val_snippets.append('%%(%s)s' % field)
@@ -1841,11 +1913,11 @@ def run_insert(link_obj=None, schema=None, table=None, values=None, returning=No
# =======================================================================
# connection handling API
# -----------------------------------------------------------------------
-def get_raw_connection(dsn=None, verbose=False, readonly=True, connection_name=None, autocommit=False):
+def get_raw_connection(verbose=False, readonly=True, connection_name=None, autocommit=False):
"""Get a raw, unadorned connection.
- this will not set any parameters such as encoding, timezone, datestyle
- - the only requirement is a valid DSN
+ - the only requirement is valid connection parameters having been passed to the connection pool
- hence it can be used for "service" connections
for verifying encodings etc
"""
@@ -1857,12 +1929,13 @@ def get_raw_connection(dsn=None, verbose=False, readonly=True, connection_name=N
)
# =======================================================================
-def get_connection(dsn=None, readonly=True, verbose=False, pooled=True, connection_name=None, autocommit=False):
+def get_connection(readonly=True, verbose=False, pooled=True, connection_name=None, autocommit=False):
return gmConnectionPool.gmConnectionPool().get_connection (
readonly = readonly,
verbose = verbose,
connection_name = connection_name,
- autocommit = autocommit
+ autocommit = autocommit,
+ pooled = pooled
)
#-----------------------------------------------------------------------
@@ -1985,7 +2058,7 @@ def sanity_check_database_settings():
cmd = "SELECT name, setting from pg_settings where name in %(settings)s"
rows, idx = run_ro_queries (
link_obj = conn,
- queries = [{'cmd': cmd, 'args': {'settings': tuple(options2check.keys())}}],
+ queries = [{'cmd': cmd, 'args': {'settings': tuple(options2check)}}],
get_col_idx = False
)
@@ -2513,6 +2586,15 @@ SELECT to_timestamp (foofoo,'YYMMDD.HH24MI') FROM (
conn = get_connection()
gmConnectionPool.log_pg_settings(curs = conn.cursor())
+ #--------------------------------------------------------------------
+ def test_get_db_fingerprint():
+ login, creds = request_login_params()
+ pool = gmConnectionPool.gmConnectionPool()
+ pool.credentials = creds
+ #conn = get_connection()
+ #print(get_db_fingerprint(conn, with_dump = True, eol = '\n'))
+ print(get_db_fingerprint(with_dump = True, eol = '\n'))
+
#--------------------------------------------------------------------
# run tests
@@ -2535,11 +2617,12 @@ SELECT to_timestamp (foofoo,'YYMMDD.HH24MI') FROM (
#test_get_index_name()
#test_set_user_language()
#test_get_schema_revision_history()
- test_run_query()
+ #test_run_query()
#test_schema_exists()
#test_get_foreign_key_names()
#test_row_locks()
#test_faulty_SQL()
- test_log_settings()
+ #test_log_settings()
+ test_get_db_fingerprint()
# ======================================================================
=====================================
server/pycommon/gmTools.py
=====================================
@@ -698,7 +698,7 @@ def old_unicode_csv_reader(unicode_csv_data, dialect=csv.excel, encoding='utf-8'
for row in csv_reader:
# decode ENCODING back to Unicode, cell by cell:
if is_dict_reader:
- for key in row.keys():
+ for key in row:
if key == default_csv_reader_rest_key:
old_data = row[key]
new_data = []
@@ -1436,8 +1436,8 @@ def compare_dict_likes(d1, d2, title1=None, title2=None):
d2 = dict(d2)
except TypeError:
pass
- keys_d1 = frozenset(d1.keys())
- keys_d2 = frozenset(d2.keys())
+ keys_d1 = list(d1)
+ keys_d2 = list(d2)
different = False
if len(keys_d1) != len(keys_d2):
_log.info('different number of keys: %s vs %s', len(keys_d1), len(keys_d2))
@@ -1487,8 +1487,8 @@ def format_dict_likes_comparison(d1, d2, title_left=None, title_right=None, left
except TypeError: pass
try: d2 = dict(d2)
except TypeError: pass
- keys_d1 = d1.keys()
- keys_d2 = d2.keys()
+ keys_d1 = list(d1)
+ keys_d2 = list(d2)
data = {}
for key in keys_d1:
data[key] = [d1[key], ' ']
@@ -1586,7 +1586,7 @@ def format_dict_like(d, relevant_keys=None, template=None, missing_key_template=
return template % d
if relevant_keys is None:
- relevant_keys = list(d.keys())
+ relevant_keys = list(d)
lines = []
if value_delimiters is None:
delim_left = ''
@@ -1636,7 +1636,7 @@ def dicts2table(dict_list, left_margin=0, eol='\n', keys2ignore=None, column_lab
d[col_label_key] = max(column_labels[dict_idx].split('\n'), key = len)
field_lengths = []
# loop over all keys in this dict
- for key in d.keys():
+ for key in d:
# ignore this key
if key in keys2ignore:
continue
=====================================
server/sql/v21-v22/fixups/v22-release_notes-fixup.sql
=====================================
@@ -17,25 +17,28 @@ INSERT INTO dem.message_inbox (
) VALUES (
(select pk from dem.staff where db_user = 'any-doc'),
(select pk_type from dem.v_inbox_item_type where type = 'memo' and category = 'administrative'),
- 'Release Notes for GNUmed 1.8.1 (database v22.12)',
- 'GNUmed 1.8.1 Release Notes:
+ 'Release Notes for GNUmed 1.8.3 (database v22.13)',
+ 'GNUmed 1.8.3 Release Notes:
- 1.8.1
+ 1.8.3
-NEW: tool: read_all_rows_of_table
+NEW: tool: fingerprint_db
-FIX: bills: failure to generate bill PDFs [thanks Marc]
-FIX: bills: failure to edit bill item date [thanks Marc]
-FIX: bills: failure to edit billable [thanks Marc]
-FIX: export area: fails to load when gm-burn.sh not found [thanks Marc]
-FIX: demographics: failure to edit type of address [thanks Marc]
-FIX: forms: failure to archive generated forms [thanks Marc]
-FIX: demographics: faulty display of patient addresses [thanks Marc]
+IMPROVED: forms: log pdflatex version
- 22.12
+FIX: meds: drug data source selection [thanks bganglia892]
+FIX: meds: ADR URL configuration [thanks bganglia892]
+FIX: vaccs: ADR URL configuration [thanks bganglia892]
+FIX: tests: LOINC import fixup [thanks bganglia892]
+FIX: db: whitespace in connection parameters [thanks kikiruz]
+FIX: startup: startup without console fails [thanks Marc]
+FIX: top panel: incorrect age-at-birthday display
+FIX: i18n: locale activation
-IMPROVED: robustify backup script
+ 22.13
+
+IMPROVED: bootstrapper logging
');
-- --------------------------------------------------------------
-select gm.log_script_insertion('v22-release_notes-fixup.sql', '22.12');
+select gm.log_script_insertion('v22-release_notes-fixup.sql', '22.13 at 1.8.3');
View it on GitLab: https://salsa.debian.org/med-team/gnumed-server/-/commit/edf8d01107dee2600f722b83486df47158255a3e
--
View it on GitLab: https://salsa.debian.org/med-team/gnumed-server/-/commit/edf8d01107dee2600f722b83486df47158255a3e
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20200724/3be7289f/attachment-0001.html>
More information about the debian-med-commit
mailing list