[Python-modules-commits] [python-vertica] 01/05: Import python-vertica_0.5.4.orig.tar.gz

Jean Baptiste Favre jbfavre-guest at moszumanska.debian.org
Sat Nov 28 21:59:19 UTC 2015


This is an automated email from the git hooks/post-receive script.

jbfavre-guest pushed a commit to branch master
in repository python-vertica.

commit d5bcebba4a896622e9d5fa9cdeb86f80e7775472
Author: Jean Baptiste Favre <debian at jbfavre.org>
Date:   Sat Nov 28 18:38:38 2015 +0100

    Import python-vertica_0.5.4.orig.tar.gz
---
 README.md                            |  70 +++++++++++++++++--
 setup.py                             |   2 +-
 vertica_python/__init__.py           |   2 +-
 vertica_python/tests/basic_tests.py  | 128 +++++++++++++++++++++++++++++++++++
 vertica_python/tests/date_tests.py   |  23 ++++++-
 vertica_python/vertica/column.py     |  29 ++++++++
 vertica_python/vertica/connection.py |  24 +++----
 vertica_python/vertica/cursor.py     |  59 ++++++++++++++--
 8 files changed, 307 insertions(+), 30 deletions(-)

diff --git a/README.md b/README.md
index 8ab27e8..7376706 100644
--- a/README.md
+++ b/README.md
@@ -53,12 +53,12 @@ Source code for vertica-python can be found at:
 **Create connection**
 
 ```python
-from vertica_python import connect
+import vertica_python
 
-conn_info = {'host': '127.0.0.1', 
-             'port': 5433, 
-             'user': 'some_user', 
-             'password': 'some_password', 
+conn_info = {'host': '127.0.0.1',
+             'port': 5433,
+             'user': 'some_user',
+             'password': 'some_password',
              'database': 'a_database'}
 
 # simple connection, with manual close
@@ -77,10 +77,12 @@ with vertica_python.connect(**conn_info) as connection:
 ```python
 cur = connection.cursor()
 cur.execute("SELECT * FROM a_table LIMIT 2")
+
 for row in cur.iterate():
     print(row)
-# {'id': 1, 'value': 'something'}
-# {'id': 2, 'value': 'something_else'}
+# [ 1, 'some text', datetime.datetime(2014, 5, 18, 6, 47, 1, 928014) ]
+# [ 2, 'something else', None ]
+
 ```
 Streaming is recommended if you want to further process each row, save the results in a non-list/dict format (e.g. Pandas DataFrame), or save the results in a file.
 
@@ -113,6 +115,7 @@ connection.close()
 
 cur = connection.cursor()
 cur.execute("SELECT * FROM a_table WHERE a = :propA b = :propB", {'propA': 1, 'propB': 'stringValue'})
+
 cur.fetchall()
 # [ [1, 'something'], [2, 'something_else'] ]
 ```
@@ -146,6 +149,59 @@ cur.copy("COPY test_copy (id, name) from stdin DELIMITER ',' ",  csv)
 Where `csv` is either a string or a file-like object (specifically, any object with a `read()` method). If using a file, the data is streamed.
 
 
+
+## Rowcount oddities
+
+vertica_python behaves a bit differently than dbapi when returning rowcounts.
+
+After a select execution, the rowcount will be -1, indicating that the row count is unknown. The rowcount value will be updated as data is streamed.
+
+```python
+cur.execute('SELECT 10 things')
+
+cur.rowcount == -1  # indicates unknown rowcount
+
+cur.fetchone()
+cur.rowcount == 1
+cur.fetchone()
+cur.rowcount == 2
+cur.fetchall()
+cur.rowcount == 10
+```
+
+After an insert/update/delete, the rowcount will be returned as a single element row:
+
+```python
+cur.execute("DELETE 3 things")
+
+cur.rowcount == -1  # indicates unknown rowcount
+cur.fetchone()[0] == 3
+```
+
+## Nextset
+
+If you execute multiple statements in a single call to execute(), you can use cursor.nextset() to retrieve all of the data.
+
+```python
+cur.execute('SELECT 1; SELECT 2;')
+
+cur.fetchone()
+# [1]
+cur.fetchone()
+# None
+
+cur.nextset()
+# True
+
+cur.fetchone()
+# [2]
+cur.fetchone()
+# None
+
+cur.nextset()
+# None
+```
+
 ## License
 
 MIT License, please see `LICENSE` for details.
diff --git a/setup.py b/setup.py
index 9c8b6e0..a014c13 100644
--- a/setup.py
+++ b/setup.py
@@ -10,7 +10,7 @@ opts = ReqOpts(None, 'git')
 # version should use the format 'x.x.x' (instead of 'vx.x.x')
 setup(
     name='vertica-python',
-    version='0.5.1',
+    version='0.5.4',
     description='A native Python client for the Vertica database.',
     author='Justin Berka, Alex Kim, Kenneth Tran',
     author_email='justin.berka at gmail.com, alex.kim at uber.com, tran at uber.com',
diff --git a/vertica_python/__init__.py b/vertica_python/__init__.py
index 63967d6..3177420 100644
--- a/vertica_python/__init__.py
+++ b/vertica_python/__init__.py
@@ -6,7 +6,7 @@ from vertica_python.vertica.connection import Connection
 # Main module for this library.
 
 # The version number of this library.
-version_info = (0, 5, 1)
+version_info = (0, 5, 4)
 
 __version__ = '.'.join(map(str, version_info))
 
diff --git a/vertica_python/tests/basic_tests.py b/vertica_python/tests/basic_tests.py
index 5991c6c..0593e40 100644
--- a/vertica_python/tests/basic_tests.py
+++ b/vertica_python/tests/basic_tests.py
@@ -32,10 +32,15 @@ class TestVerticaPython(unittest.TestCase):
         
         cur.execute(""" INSERT INTO vertica_python_unit_test (a, b) VALUES (1, 'aa'); commit; """)
         cur.execute("SELECT a, b from vertica_python_unit_test WHERE a = 1")
+
+        # unknown rowcount
+        assert cur.rowcount == -1
+
         res = cur.fetchall()
         assert 1 == len(res)
         assert 1 == res[0][0]
         assert 'aa' == res[0][1]
+        assert cur.rowcount == 1
 
     def test_multi_inserts_and_transaction(self):
 
@@ -88,6 +93,30 @@ class TestVerticaPython(unittest.TestCase):
         assert 1 == len(res)
 
 
+    def test_delete(self):
+
+        conn = vertica_python.connect(**conn_info)
+        cur = conn.cursor()
+        init_table(cur)
+
+        cur.execute(""" INSERT INTO vertica_python_unit_test (a, b) VALUES (5, 'cc') """)
+        conn.commit()
+
+        cur.execute(""" DELETE from vertica_python_unit_test WHERE a = 5 """)
+
+        # validate delete count
+        assert cur.rowcount == -1
+        res = cur.fetchone()
+        assert 1 == len(res)
+        assert 1 == res[0]
+
+        conn.commit()
+
+        cur.execute("SELECT a, b from vertica_python_unit_test WHERE a = 5")
+        res = cur.fetchall()
+        assert 0 == len(res)
+
+
     def test_update(self):
 
         conn = vertica_python.connect(**conn_info)
@@ -95,9 +124,22 @@ class TestVerticaPython(unittest.TestCase):
         init_table(cur)
     
         cur.execute(""" INSERT INTO vertica_python_unit_test (a, b) VALUES (5, 'cc') """)
+
+        # validate insert count
+        res = cur.fetchone()
+        assert 1 == len(res)
+        assert 1 == res[0]
+
         conn.commit()
     
         cur.execute(""" UPDATE vertica_python_unit_test SET b = 'ff' WHERE a = 5 """)
+
+        # validate update count
+        assert cur.rowcount == -1
+        res = cur.fetchone()
+        assert 1 == len(res)
+        assert 1 == res[0]
+
         conn.commit()
     
         cur.execute("SELECT a, b from vertica_python_unit_test WHERE a = 5")
@@ -281,3 +323,89 @@ class TestVerticaPython(unittest.TestCase):
         cur.execute("SELECT a, b from vertica_python_unit_test")
         res = cur.fetchall()
         assert 1 == len(res)
+
+    # unit test for #78
+    def test_copy_with_data_in_buffer(self):
+
+        conn = vertica_python.connect(**conn_info)
+        cur = conn.cursor()
+        init_table(cur)
+
+        cur.execute("select 1;")
+        cur.fetchall()
+
+        # Current status: CommandComplete
+
+        copy_sql = """COPY vertica_python_unit_test (a, b)
+                     FROM STDIN
+                     DELIMITER '|'
+                     NULL AS 'None'"""
+
+        data = """1|name1
+        2|name2"""
+
+        cur.copy(copy_sql, data)
+        cur.execute("select 1;") # will raise QueryError here
+
+        conn.close()
+
+    # unit test for #74
+    def test_nextset(self):
+
+        conn = vertica_python.connect(**conn_info)
+        cur = conn.cursor()
+        init_table(cur)
+
+        cur.execute("select 1; select 2;")
+        res = cur.fetchall()
+
+        assert 1 == len(res)
+        assert 1 == res[0][0]
+        assert cur.fetchone() is None
+
+        assert cur.nextset() == True
+
+        res = cur.fetchall()
+        assert 1 == len(res)
+        assert 2 == res[0][0]
+        assert cur.fetchone() is None
+
+        assert cur.nextset() is None
+
+    # unit test for #74
+    def test_nextset_with_delete(self):
+
+        conn = vertica_python.connect(**conn_info)
+        cur = conn.cursor()
+        init_table(cur)
+
+        # insert data
+        cur.execute(""" INSERT INTO vertica_python_unit_test (a, b) VALUES (1, 'aa') """)
+        cur.execute(""" INSERT INTO vertica_python_unit_test (a, b) VALUES (2, 'bb') """)
+        conn.commit()
+
+        cur.execute("""select * from vertica_python_unit_test;
+                    delete from vertica_python_unit_test;
+                    select * from vertica_python_unit_test;
+                    """)
+
+        # check first select results
+        res = cur.fetchall()
+        assert 2 == len(res)
+        assert cur.fetchone() is None
+
+        # check delete results
+        assert cur.nextset() == True
+        res = cur.fetchall()
+        assert 1 == len(res)
+        assert 2 == res[0][0]
+        assert cur.fetchone() is None
+
+        # check second select results
+        assert cur.nextset() == True
+        res = cur.fetchall()
+        assert 0 == len(res)
+        assert cur.fetchone() is None
+
+        # no more data sets
+        assert cur.nextset() is None
diff --git a/vertica_python/tests/date_tests.py b/vertica_python/tests/date_tests.py
index 91b1a4d..90e4486 100644
--- a/vertica_python/tests/date_tests.py
+++ b/vertica_python/tests/date_tests.py
@@ -1,6 +1,7 @@
-from datetime import date
+from datetime import date, datetime
 from test_commons import *
 from vertica_python import errors
+from vertica_python.vertica.column import timestamp_parse
 
 
 class DateParsingTestCase(VerticaTestCase):
@@ -48,3 +49,23 @@ class DateParsingTestCase(VerticaTestCase):
             self.fail("Expected to see NotSupportedError when Before Christ date is encountered. Got: " + str(res))
         except errors.NotSupportedError:
             pass
+
+
+class TimestampParsingTestCase(VerticaTestCase):
+    """Verify timestamp parsing works properly."""
+
+
+    def test_timestamp_parser(self):
+        parsed_timestamp = timestamp_parse('1841-05-05 22:07:58')
+        # Assert parser default to strptime
+        self.assertEqual(datetime(year=1841, month=5, day=5, hour=22, minute=7, second=58), parsed_timestamp)
+
+    def test_timestamp_with_year_over_9999(self):
+        parsed_timestamp = timestamp_parse('44841-05-05 22:07:58')
+        # Assert year was truncated properly
+        self.assertEqual(datetime(year=4841, month=5, day=5, hour=22, minute=7, second=58), parsed_timestamp)
+
+    def test_timestamp_with_year_over_9999_and_ms(self):
+        parsed_timestamp = timestamp_parse('124841-05-05 22:07:58.000003')
+        # Assert year was truncated properly
+        self.assertEqual(datetime(year=4841, month=5, day=5, hour=22, minute=7, second=58, microsecond=3), parsed_timestamp)
diff --git a/vertica_python/vertica/column.py b/vertica_python/vertica/column.py
index 1c6a258..871da65 100644
--- a/vertica_python/vertica/column.py
+++ b/vertica_python/vertica/column.py
@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 
 from collections import namedtuple
+import re
 
 from decimal import Decimal
 from datetime import date
@@ -10,6 +11,8 @@ from vertica_python import errors
 
 import pytz
 
+years_re = re.compile(r'^([0-9]+)-')
+
 
 # these methods are bad...
 #
@@ -30,11 +33,33 @@ import pytz
 # timestamptz type stores: 2013-01-01 05:00:00.01+00
 #       select t AT TIMEZONE 'America/New_York' returns: 2012-12-31 19:00:00.01
 def timestamp_parse(s):
+    try:
+        dt = _timestamp_parse(s)
+    except ValueError:
+        # Value error, year might be over 9999
+        year_match = years_re.match(s)
+        if year_match:
+            year = year_match.groups()[0]
+            dt = _timestamp_parse_without_year(s[len(year) + 1:])
+            dt = dt.replace(year=int(year) % 10000)
+        else:
+            raise errors.DataError('Timestamp value not supported: %s' % s)
+
+    return dt
+
+
+def _timestamp_parse(s):
     if len(s) == 19:
         return datetime.strptime(s, '%Y-%m-%d %H:%M:%S')
     return datetime.strptime(s, '%Y-%m-%d %H:%M:%S.%f')
 
 
+def _timestamp_parse_without_year(s):
+    if len(s) == 14:
+        return datetime.strptime(s, '%m-%d %H:%M:%S')
+    return datetime.strptime(s, '%m-%d %H:%M:%S.%f')
+
+
 def timestamp_tz_parse(s):
     # if timezome is simply UTC...
     if s.endswith('+00'):
@@ -104,6 +129,10 @@ class Column(object):
         if self.type_code == 115:
             self.type_code = 9
 
+        # Mark type_code as unspecified if not within known data types
+        if self.type_code >= len(self.DATA_TYPE_CONVERSIONS):
+            self.type_code = 0
+
         #self.props = ColumnTuple(col['name'], col['data_type_oid'], None, col['data_type_size'], None, None, None)
         self.props = ColumnTuple(col['name'], self.type_code, None, col['data_type_size'], None, None, None)
 
diff --git a/vertica_python/vertica/connection.py b/vertica_python/vertica/connection.py
index 846ea08..8e2e12c 100644
--- a/vertica_python/vertica/connection.py
+++ b/vertica_python/vertica/connection.py
@@ -31,7 +31,7 @@ class Connection(object):
         self._cursor = Cursor(self, None)
         self.options.setdefault('port', 5433)
         self.options.setdefault('read_timeout', 600)
-        self.boot_connection()
+        self.startup_connection()
 
     def __enter__(self):
         return self
@@ -152,7 +152,10 @@ class Connection(object):
 
         except Exception, e:
             self.close_socket()
-            raise errors.ConnectionError(e.message)
+            if e.message == 'unsupported authentication method: 9':
+                raise errors.ConnectionError('Error during authentication. Your password might be expired.')
+            else:
+                raise errors.ConnectionError(e.message)
 
     def close_socket(self):
         try:
@@ -163,11 +166,7 @@ class Connection(object):
 
     def reset_connection(self):
         self.close()
-        self.boot_connection()
-
-    def boot_connection(self):
         self.startup_connection()
-        self.initialize_connection()
 
     def read_message(self):
         try:
@@ -206,6 +205,10 @@ class Connection(object):
         elif isinstance(message, messages.ReadyForQuery):
             self.transaction_status = message.transaction_status
         elif isinstance(message, messages.CommandComplete):
+            # TODO: im not ever seeing this actually returned by vertica...
+            # if vertica returns a row count, set the rowcount attribute in cursor
+            #if hasattr(message, 'rows'):
+            #    self.cursor.rowcount = message.rows
             pass
         elif isinstance(message, messages.CopyInResponse):
             pass
@@ -259,12 +262,3 @@ class Connection(object):
 
             if isinstance(message, messages.ReadyForQuery):
                 break
-
-    def initialize_connection(self):
-        if self.options.get('search_path') is not None:
-            self.query("SET SEARCH_PATH TO {0}".format(self.options['search_path']))
-        if self.options.get('role') is not None:
-            self.query("SET ROLE {0}".format(self.options['role']))
-
-# if self.options.get('interruptable'):
-#            self.session_id = self.query("SELECT session_id FROM v_monitor.current_session").the_value()
diff --git a/vertica_python/vertica/cursor.py b/vertica_python/vertica/cursor.py
index dc987fe..391c2f9 100644
--- a/vertica_python/vertica/cursor.py
+++ b/vertica_python/vertica/cursor.py
@@ -76,7 +76,7 @@ class Cursor(object):
             else:
                 raise errors.Error("Argument 'parameters' must be dict or tuple")
 
-        self.rowcount = 0
+        self.rowcount = -1
 
         self.connection.write(messages.Query(operation))
 
@@ -89,19 +89,28 @@ class Cursor(object):
                 raise errors.QueryError.from_error_response(message, operation)
             elif isinstance(message, messages.RowDescription):
                 self.description = map(lambda fd: Column(fd), message.fields)
-            elif (isinstance(message, messages.DataRow)
-                   or isinstance(message, messages.ReadyForQuery)):
+            elif isinstance(message, messages.DataRow):
+                break
+            elif isinstance(message, messages.ReadyForQuery):
                 break
             else:
                 self.connection.process_message(message)
 
     def fetchone(self):
         if isinstance(self._message, messages.DataRow):
-            self.rowcount += 1
+            if self.rowcount == -1:
+                self.rowcount = 1
+            else:
+                self.rowcount += 1
+
             row = self.row_formatter(self._message)
             # fetch next message
             self._message = self.connection.read_message()
             return row
+        elif isinstance(self._message, messages.ReadyForQuery):
+            return None
+        elif isinstance(self._message, messages.CommandComplete):
+            return None
         else:
             self.connection.process_message(self._message)
 
@@ -127,6 +136,30 @@ class Cursor(object):
     def fetchall(self):
         return list(self.iterate())
 
+    def nextset(self):
+        # skip any data for this set if exists
+        self.flush_to_command_complete()
+
+        if self._message is None:
+            return None
+        elif isinstance(self._message, messages.CommandComplete):
+            # there might be another set, read next message to find out
+            self._message = self.connection.read_message()
+            if isinstance(self._message, messages.RowDescription):
+                # next row will be either a DataRow or CommandComplete
+                self._message = self.connection.read_message()
+                return True
+            elif isinstance(self._message, messages.ReadyForQuery):
+                return None
+            else:
+                raise errors.Error('Unexpected nextset() state after CommandComplete: ' + str(self._message))
+        elif isinstance(self._message, messages.ReadyForQuery):
+            # no more sets left to be read
+            return None
+        else:
+            raise errors.Error('Unexpected nextset() state: ' + str(self._message))
+
+
     def setinputsizes(self):
         pass
 
@@ -149,6 +182,20 @@ class Cursor(object):
                 self._message = message
                 break
 
+    def flush_to_command_complete(self):
+        # if the last message isnt empty or CommandComplete, read messages until it is
+        if(self._message is None
+           or isinstance(self._message, messages.ReadyForQuery)
+           or isinstance(self._message, messages.CommandComplete)):
+            return
+
+        while True:
+            message = self.connection.read_message()
+            if isinstance(message, messages.CommandComplete):
+                self._message = message
+                break
+
+
     # example:
     #
     # with open("/tmp/file.csv", "rb") as fs:
@@ -160,6 +207,8 @@ class Cursor(object):
         if self.closed():
             raise errors.Error('Cursor is closed')
 
+        self.flush_to_query_ready()
+
         self.connection.write(messages.Query(sql))
 
         while True:
@@ -205,4 +254,4 @@ class Cursor(object):
 
     def format_row_as_array(self, row_data):
         return [self.description[idx].convert(value)
-                for idx, value in enumerate(row_data.values)]
\ No newline at end of file
+                for idx, value in enumerate(row_data.values)]

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/python-vertica.git



More information about the Python-modules-commits mailing list