[Python-modules-commits] [python-vertica] 01/05: Import python-vertica_0.5.4.orig.tar.gz
Jean Baptiste Favre
jbfavre-guest at moszumanska.debian.org
Sat Nov 28 21:59:19 UTC 2015
This is an automated email from the git hooks/post-receive script.
jbfavre-guest pushed a commit to branch master
in repository python-vertica.
commit d5bcebba4a896622e9d5fa9cdeb86f80e7775472
Author: Jean Baptiste Favre <debian at jbfavre.org>
Date: Sat Nov 28 18:38:38 2015 +0100
Import python-vertica_0.5.4.orig.tar.gz
---
README.md | 70 +++++++++++++++++--
setup.py | 2 +-
vertica_python/__init__.py | 2 +-
vertica_python/tests/basic_tests.py | 128 +++++++++++++++++++++++++++++++++++
vertica_python/tests/date_tests.py | 23 ++++++-
vertica_python/vertica/column.py | 29 ++++++++
vertica_python/vertica/connection.py | 24 +++----
vertica_python/vertica/cursor.py | 59 ++++++++++++++--
8 files changed, 307 insertions(+), 30 deletions(-)
diff --git a/README.md b/README.md
index 8ab27e8..7376706 100644
--- a/README.md
+++ b/README.md
@@ -53,12 +53,12 @@ Source code for vertica-python can be found at:
**Create connection**
```python
-from vertica_python import connect
+import vertica_python
-conn_info = {'host': '127.0.0.1',
- 'port': 5433,
- 'user': 'some_user',
- 'password': 'some_password',
+conn_info = {'host': '127.0.0.1',
+ 'port': 5433,
+ 'user': 'some_user',
+ 'password': 'some_password',
'database': 'a_database'}
# simple connection, with manual close
@@ -77,10 +77,12 @@ with vertica_python.connect(**conn_info) as connection:
```python
cur = connection.cursor()
cur.execute("SELECT * FROM a_table LIMIT 2")
+
for row in cur.iterate():
print(row)
-# {'id': 1, 'value': 'something'}
-# {'id': 2, 'value': 'something_else'}
+# [ 1, 'some text', datetime.datetime(2014, 5, 18, 6, 47, 1, 928014) ]
+# [ 2, 'something else', None ]
+
```
Streaming is recommended if you want to further process each row, save the results in a non-list/dict format (e.g. Pandas DataFrame), or save the results in a file.
@@ -113,6 +115,7 @@ connection.close()
cur = connection.cursor()
cur.execute("SELECT * FROM a_table WHERE a = :propA b = :propB", {'propA': 1, 'propB': 'stringValue'})
+
cur.fetchall()
# [ [1, 'something'], [2, 'something_else'] ]
```
@@ -146,6 +149,59 @@ cur.copy("COPY test_copy (id, name) from stdin DELIMITER ',' ", csv)
Where `csv` is either a string or a file-like object (specifically, any object with a `read()` method). If using a file, the data is streamed.
+
+## Rowcount oddities
+
+vertica_python behaves a bit differently than dbapi when returning rowcounts.
+
+After a select execution, the rowcount will be -1, indicating that the row count is unknown. The rowcount value will be updated as data is streamed.
+
+```python
+cur.execute('SELECT 10 things')
+
+cur.rowcount == -1 # indicates unknown rowcount
+
+cur.fetchone()
+cur.rowcount == 1
+cur.fetchone()
+cur.rowcount == 2
+cur.fetchall()
+cur.rowcount == 10
+```
+
+After an insert/update/delete, the rowcount will be returned as a single element row:
+
+```python
+cur.execute("DELETE 3 things")
+
+cur.rowcount == -1 # indicates unknown rowcount
+cur.fetchone()[0] == 3
+```
+
+## Nextset
+
+If you execute multiple statements in a single call to execute(), you can use cursor.nextset() to retrieve all of the data.
+
+```python
+cur.execute('SELECT 1; SELECT 2;')
+
+cur.fetchone()
+# [1]
+cur.fetchone()
+# None
+
+cur.nextset()
+# True
+
+cur.fetchone()
+# [2]
+cur.fetchone()
+# None
+
+cur.nextset()
+# None
+```
+
## License
MIT License, please see `LICENSE` for details.
diff --git a/setup.py b/setup.py
index 9c8b6e0..a014c13 100644
--- a/setup.py
+++ b/setup.py
@@ -10,7 +10,7 @@ opts = ReqOpts(None, 'git')
# version should use the format 'x.x.x' (instead of 'vx.x.x')
setup(
name='vertica-python',
- version='0.5.1',
+ version='0.5.4',
description='A native Python client for the Vertica database.',
author='Justin Berka, Alex Kim, Kenneth Tran',
author_email='justin.berka at gmail.com, alex.kim at uber.com, tran at uber.com',
diff --git a/vertica_python/__init__.py b/vertica_python/__init__.py
index 63967d6..3177420 100644
--- a/vertica_python/__init__.py
+++ b/vertica_python/__init__.py
@@ -6,7 +6,7 @@ from vertica_python.vertica.connection import Connection
# Main module for this library.
# The version number of this library.
-version_info = (0, 5, 1)
+version_info = (0, 5, 4)
__version__ = '.'.join(map(str, version_info))
diff --git a/vertica_python/tests/basic_tests.py b/vertica_python/tests/basic_tests.py
index 5991c6c..0593e40 100644
--- a/vertica_python/tests/basic_tests.py
+++ b/vertica_python/tests/basic_tests.py
@@ -32,10 +32,15 @@ class TestVerticaPython(unittest.TestCase):
cur.execute(""" INSERT INTO vertica_python_unit_test (a, b) VALUES (1, 'aa'); commit; """)
cur.execute("SELECT a, b from vertica_python_unit_test WHERE a = 1")
+
+ # unknown rowcount
+ assert cur.rowcount == -1
+
res = cur.fetchall()
assert 1 == len(res)
assert 1 == res[0][0]
assert 'aa' == res[0][1]
+ assert cur.rowcount == 1
def test_multi_inserts_and_transaction(self):
@@ -88,6 +93,30 @@ class TestVerticaPython(unittest.TestCase):
assert 1 == len(res)
+ def test_delete(self):
+
+ conn = vertica_python.connect(**conn_info)
+ cur = conn.cursor()
+ init_table(cur)
+
+ cur.execute(""" INSERT INTO vertica_python_unit_test (a, b) VALUES (5, 'cc') """)
+ conn.commit()
+
+ cur.execute(""" DELETE from vertica_python_unit_test WHERE a = 5 """)
+
+ # validate delete count
+ assert cur.rowcount == -1
+ res = cur.fetchone()
+ assert 1 == len(res)
+ assert 1 == res[0]
+
+ conn.commit()
+
+ cur.execute("SELECT a, b from vertica_python_unit_test WHERE a = 5")
+ res = cur.fetchall()
+ assert 0 == len(res)
+
+
def test_update(self):
conn = vertica_python.connect(**conn_info)
@@ -95,9 +124,22 @@ class TestVerticaPython(unittest.TestCase):
init_table(cur)
cur.execute(""" INSERT INTO vertica_python_unit_test (a, b) VALUES (5, 'cc') """)
+
+ # validate insert count
+ res = cur.fetchone()
+ assert 1 == len(res)
+ assert 1 == res[0]
+
conn.commit()
cur.execute(""" UPDATE vertica_python_unit_test SET b = 'ff' WHERE a = 5 """)
+
+ # validate update count
+ assert cur.rowcount == -1
+ res = cur.fetchone()
+ assert 1 == len(res)
+ assert 1 == res[0]
+
conn.commit()
cur.execute("SELECT a, b from vertica_python_unit_test WHERE a = 5")
@@ -281,3 +323,89 @@ class TestVerticaPython(unittest.TestCase):
cur.execute("SELECT a, b from vertica_python_unit_test")
res = cur.fetchall()
assert 1 == len(res)
+
+ # unit test for #78
+ def test_copy_with_data_in_buffer(self):
+
+ conn = vertica_python.connect(**conn_info)
+ cur = conn.cursor()
+ init_table(cur)
+
+ cur.execute("select 1;")
+ cur.fetchall()
+
+ # Current status: CommandComplete
+
+ copy_sql = """COPY vertica_python_unit_test (a, b)
+ FROM STDIN
+ DELIMITER '|'
+ NULL AS 'None'"""
+
+ data = """1|name1
+ 2|name2"""
+
+ cur.copy(copy_sql, data)
+ cur.execute("select 1;") # will raise QueryError here
+
+ conn.close()
+
+ # unit test for #74
+ def test_nextset(self):
+
+ conn = vertica_python.connect(**conn_info)
+ cur = conn.cursor()
+ init_table(cur)
+
+ cur.execute("select 1; select 2;")
+ res = cur.fetchall()
+
+ assert 1 == len(res)
+ assert 1 == res[0][0]
+ assert cur.fetchone() is None
+
+ assert cur.nextset() == True
+
+ res = cur.fetchall()
+ assert 1 == len(res)
+ assert 2 == res[0][0]
+ assert cur.fetchone() is None
+
+ assert cur.nextset() is None
+
+ # unit test for #74
+ def test_nextset_with_delete(self):
+
+ conn = vertica_python.connect(**conn_info)
+ cur = conn.cursor()
+ init_table(cur)
+
+ # insert data
+ cur.execute(""" INSERT INTO vertica_python_unit_test (a, b) VALUES (1, 'aa') """)
+ cur.execute(""" INSERT INTO vertica_python_unit_test (a, b) VALUES (2, 'bb') """)
+ conn.commit()
+
+ cur.execute("""select * from vertica_python_unit_test;
+ delete from vertica_python_unit_test;
+ select * from vertica_python_unit_test;
+ """)
+
+ # check first select results
+ res = cur.fetchall()
+ assert 2 == len(res)
+ assert cur.fetchone() is None
+
+ # check delete results
+ assert cur.nextset() == True
+ res = cur.fetchall()
+ assert 1 == len(res)
+ assert 2 == res[0][0]
+ assert cur.fetchone() is None
+
+ # check second select results
+ assert cur.nextset() == True
+ res = cur.fetchall()
+ assert 0 == len(res)
+ assert cur.fetchone() is None
+
+ # no more data sets
+ assert cur.nextset() is None
diff --git a/vertica_python/tests/date_tests.py b/vertica_python/tests/date_tests.py
index 91b1a4d..90e4486 100644
--- a/vertica_python/tests/date_tests.py
+++ b/vertica_python/tests/date_tests.py
@@ -1,6 +1,7 @@
-from datetime import date
+from datetime import date, datetime
from test_commons import *
from vertica_python import errors
+from vertica_python.vertica.column import timestamp_parse
class DateParsingTestCase(VerticaTestCase):
@@ -48,3 +49,23 @@ class DateParsingTestCase(VerticaTestCase):
self.fail("Expected to see NotSupportedError when Before Christ date is encountered. Got: " + str(res))
except errors.NotSupportedError:
pass
+
+
+class TimestampParsingTestCase(VerticaTestCase):
+ """Verify timestamp parsing works properly."""
+
+
+ def test_timestamp_parser(self):
+ parsed_timestamp = timestamp_parse('1841-05-05 22:07:58')
+ # Assert parser default to strptime
+ self.assertEqual(datetime(year=1841, month=5, day=5, hour=22, minute=7, second=58), parsed_timestamp)
+
+ def test_timestamp_with_year_over_9999(self):
+ parsed_timestamp = timestamp_parse('44841-05-05 22:07:58')
+ # Assert year was truncated properly
+ self.assertEqual(datetime(year=4841, month=5, day=5, hour=22, minute=7, second=58), parsed_timestamp)
+
+ def test_timestamp_with_year_over_9999_and_ms(self):
+ parsed_timestamp = timestamp_parse('124841-05-05 22:07:58.000003')
+ # Assert year was truncated properly
+ self.assertEqual(datetime(year=4841, month=5, day=5, hour=22, minute=7, second=58, microsecond=3), parsed_timestamp)
diff --git a/vertica_python/vertica/column.py b/vertica_python/vertica/column.py
index 1c6a258..871da65 100644
--- a/vertica_python/vertica/column.py
+++ b/vertica_python/vertica/column.py
@@ -1,6 +1,7 @@
from __future__ import absolute_import
from collections import namedtuple
+import re
from decimal import Decimal
from datetime import date
@@ -10,6 +11,8 @@ from vertica_python import errors
import pytz
+years_re = re.compile(r'^([0-9]+)-')
+
# these methods are bad...
#
@@ -30,11 +33,33 @@ import pytz
# timestamptz type stores: 2013-01-01 05:00:00.01+00
# select t AT TIMEZONE 'America/New_York' returns: 2012-12-31 19:00:00.01
def timestamp_parse(s):
+ try:
+ dt = _timestamp_parse(s)
+ except ValueError:
+ # Value error, year might be over 9999
+ year_match = years_re.match(s)
+ if year_match:
+ year = year_match.groups()[0]
+ dt = _timestamp_parse_without_year(s[len(year) + 1:])
+ dt = dt.replace(year=int(year) % 10000)
+ else:
+ raise errors.DataError('Timestamp value not supported: %s' % s)
+
+ return dt
+
+
+def _timestamp_parse(s):
if len(s) == 19:
return datetime.strptime(s, '%Y-%m-%d %H:%M:%S')
return datetime.strptime(s, '%Y-%m-%d %H:%M:%S.%f')
+def _timestamp_parse_without_year(s):
+ if len(s) == 14:
+ return datetime.strptime(s, '%m-%d %H:%M:%S')
+ return datetime.strptime(s, '%m-%d %H:%M:%S.%f')
+
+
def timestamp_tz_parse(s):
# if timezome is simply UTC...
if s.endswith('+00'):
@@ -104,6 +129,10 @@ class Column(object):
if self.type_code == 115:
self.type_code = 9
+ # Mark type_code as unspecified if not within known data types
+ if self.type_code >= len(self.DATA_TYPE_CONVERSIONS):
+ self.type_code = 0
+
#self.props = ColumnTuple(col['name'], col['data_type_oid'], None, col['data_type_size'], None, None, None)
self.props = ColumnTuple(col['name'], self.type_code, None, col['data_type_size'], None, None, None)
diff --git a/vertica_python/vertica/connection.py b/vertica_python/vertica/connection.py
index 846ea08..8e2e12c 100644
--- a/vertica_python/vertica/connection.py
+++ b/vertica_python/vertica/connection.py
@@ -31,7 +31,7 @@ class Connection(object):
self._cursor = Cursor(self, None)
self.options.setdefault('port', 5433)
self.options.setdefault('read_timeout', 600)
- self.boot_connection()
+ self.startup_connection()
def __enter__(self):
return self
@@ -152,7 +152,10 @@ class Connection(object):
except Exception, e:
self.close_socket()
- raise errors.ConnectionError(e.message)
+ if e.message == 'unsupported authentication method: 9':
+ raise errors.ConnectionError('Error during authentication. Your password might be expired.')
+ else:
+ raise errors.ConnectionError(e.message)
def close_socket(self):
try:
@@ -163,11 +166,7 @@ class Connection(object):
def reset_connection(self):
self.close()
- self.boot_connection()
-
- def boot_connection(self):
self.startup_connection()
- self.initialize_connection()
def read_message(self):
try:
@@ -206,6 +205,10 @@ class Connection(object):
elif isinstance(message, messages.ReadyForQuery):
self.transaction_status = message.transaction_status
elif isinstance(message, messages.CommandComplete):
+ # TODO: im not ever seeing this actually returned by vertica...
+ # if vertica returns a row count, set the rowcount attribute in cursor
+ #if hasattr(message, 'rows'):
+ # self.cursor.rowcount = message.rows
pass
elif isinstance(message, messages.CopyInResponse):
pass
@@ -259,12 +262,3 @@ class Connection(object):
if isinstance(message, messages.ReadyForQuery):
break
-
- def initialize_connection(self):
- if self.options.get('search_path') is not None:
- self.query("SET SEARCH_PATH TO {0}".format(self.options['search_path']))
- if self.options.get('role') is not None:
- self.query("SET ROLE {0}".format(self.options['role']))
-
-# if self.options.get('interruptable'):
-# self.session_id = self.query("SELECT session_id FROM v_monitor.current_session").the_value()
diff --git a/vertica_python/vertica/cursor.py b/vertica_python/vertica/cursor.py
index dc987fe..391c2f9 100644
--- a/vertica_python/vertica/cursor.py
+++ b/vertica_python/vertica/cursor.py
@@ -76,7 +76,7 @@ class Cursor(object):
else:
raise errors.Error("Argument 'parameters' must be dict or tuple")
- self.rowcount = 0
+ self.rowcount = -1
self.connection.write(messages.Query(operation))
@@ -89,19 +89,28 @@ class Cursor(object):
raise errors.QueryError.from_error_response(message, operation)
elif isinstance(message, messages.RowDescription):
self.description = map(lambda fd: Column(fd), message.fields)
- elif (isinstance(message, messages.DataRow)
- or isinstance(message, messages.ReadyForQuery)):
+ elif isinstance(message, messages.DataRow):
+ break
+ elif isinstance(message, messages.ReadyForQuery):
break
else:
self.connection.process_message(message)
def fetchone(self):
if isinstance(self._message, messages.DataRow):
- self.rowcount += 1
+ if self.rowcount == -1:
+ self.rowcount = 1
+ else:
+ self.rowcount += 1
+
row = self.row_formatter(self._message)
# fetch next message
self._message = self.connection.read_message()
return row
+ elif isinstance(self._message, messages.ReadyForQuery):
+ return None
+ elif isinstance(self._message, messages.CommandComplete):
+ return None
else:
self.connection.process_message(self._message)
@@ -127,6 +136,30 @@ class Cursor(object):
def fetchall(self):
return list(self.iterate())
+ def nextset(self):
+ # skip any data for this set if exists
+ self.flush_to_command_complete()
+
+ if self._message is None:
+ return None
+ elif isinstance(self._message, messages.CommandComplete):
+ # there might be another set, read next message to find out
+ self._message = self.connection.read_message()
+ if isinstance(self._message, messages.RowDescription):
+ # next row will be either a DataRow or CommandComplete
+ self._message = self.connection.read_message()
+ return True
+ elif isinstance(self._message, messages.ReadyForQuery):
+ return None
+ else:
+ raise errors.Error('Unexpected nextset() state after CommandComplete: ' + str(self._message))
+ elif isinstance(self._message, messages.ReadyForQuery):
+ # no more sets left to be read
+ return None
+ else:
+ raise errors.Error('Unexpected nextset() state: ' + str(self._message))
+
+
def setinputsizes(self):
pass
@@ -149,6 +182,20 @@ class Cursor(object):
self._message = message
break
+ def flush_to_command_complete(self):
+ # if the last message isnt empty or CommandComplete, read messages until it is
+ if(self._message is None
+ or isinstance(self._message, messages.ReadyForQuery)
+ or isinstance(self._message, messages.CommandComplete)):
+ return
+
+ while True:
+ message = self.connection.read_message()
+ if isinstance(message, messages.CommandComplete):
+ self._message = message
+ break
+
+
# example:
#
# with open("/tmp/file.csv", "rb") as fs:
@@ -160,6 +207,8 @@ class Cursor(object):
if self.closed():
raise errors.Error('Cursor is closed')
+ self.flush_to_query_ready()
+
self.connection.write(messages.Query(sql))
while True:
@@ -205,4 +254,4 @@ class Cursor(object):
def format_row_as_array(self, row_data):
return [self.description[idx].convert(value)
- for idx, value in enumerate(row_data.values)]
\ No newline at end of file
+ for idx, value in enumerate(row_data.values)]
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/python-vertica.git
More information about the Python-modules-commits
mailing list