[med-svn] [python-screed] 01/05: New upstream version 1.0
Michael Crusoe
misterc-guest at moszumanska.debian.org
Fri Jun 23 15:18:47 UTC 2017
This is an automated email from the git hooks/post-receive script.
misterc-guest pushed a commit to branch master
in repository python-screed.
commit 2d03733c76decd55a353b05c707c209b21a7233d
Author: Michael R. Crusoe <michael.crusoe at gmail.com>
Date: Fri Jun 23 06:25:43 2017 -0700
New upstream version 1.0
---
.coveragerc | 2 +
.github/CONTRIBUTING.md | 3 +
.github/PULL_REQUEST_TEMPLATE.md | 9 +
.travis.yml | 3 +-
CHANGELOG.md | 15 +
Makefile | 21 +-
README.md | 2 +-
TODO | 16 -
benchmarks/faGen.py | 1 +
benchmarks/fqGen.py | 1 +
benchmarks/fqToFaConvert.py | 1 +
benchmarks/screedCreateTimeit.py | 1 +
benchmarks/screedTimeit.py | 1 +
benchmarks/screedTimeit1M.py | 1 +
bigtests/__init__.py | 269 ++--
doc/LICENSE.rst | 28 +-
doc/conf.py | 158 ++-
doc/dev/coding-guidelines-and-review.rst | 12 +-
doc/dev/index.rst | 3 +-
doc/dev/parsers.rst | 167 +++
doc/dev/release-checklist.rst | 36 +-
doc/index.rst | 10 +-
doc/release-notes/RELEASE-1.0.md | 52 +
doc/release-notes/index.rst | 4 +-
doc/requirements.txt | 1 +
doc/screed.rst | 473 ++-----
get_version.py | 1 +
ChangeLog => legacy/ChangeLog | 56 +-
jenkins-build.sh => legacy/jenkins-build.sh | 2 +-
screed/DBConstants.py | 2 +-
screed/__init__.py | 12 +-
screed/__main__.py | 52 +
screed/_version.py | 489 +++++--
screed/conversion.py | 12 +-
screed/createscreed.py | 51 +-
screed/dna.py | 9 +-
screed/dump_fasta.py | 33 +
screed/dump_fastq.py | 31 +
screed/dump_to_fasta.py | 25 -
screed/dump_to_fastq.py | 25 -
screed/fadbm.py | 19 -
screed/fasta.py | 17 +-
screed/fastq.py | 17 +-
screed/fqdbm.py | 22 -
screed/hava.py | 3 +-
screed/openscreed.py | 42 +-
screed/pygr_api.py | 2 +-
screed/screedRecord.py | 118 +-
screed/seqparse.py | 6 +-
screed/tests/__main__.py | 17 +-
screed/tests/havaGen.py | 2 +-
screed/tests/screed_tst_utils.py | 1 -
screed/tests/test_attriberror.py | 63 +
screed/tests/test_db.py | 76 ++
screed/tests/test_fasta.py | 46 +-
screed/tests/test_fastq.py | 55 +-
screed/tests/test_nodb.py | 33 -
screed/tests/test_open.py | 13 +-
screed/tests/test_open_cm.py | 13 +-
screed/tests/test_pygr_api.py | 25 +-
screed/tests/test_record.py | 34 +
screed/tests/test_shell.py | 108 +-
screed/tests/test_streaming.py | 6 +-
screed/utils.py | 3 +
setup.cfg | 16 +
setup.py | 37 +-
tox.ini | 16 +-
versioneer.py | 1904 +++++++++++++++++++--------
68 files changed, 3163 insertions(+), 1641 deletions(-)
diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 0000000..894fb09
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,2 @@
+[run]
+omit = screed/tests/*
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
new file mode 100644
index 0000000..1fd8729
--- /dev/null
+++ b/.github/CONTRIBUTING.md
@@ -0,0 +1,3 @@
+See [our development docs](https://screed.readthedocs.io/en/latest/dev/).
+
+Be sure to copy and paste the [checklist](https://screed.readthedocs.io/en/latest/dev/coding-guidelines-and-review.html#checklist) in the Pull-Request comment
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
new file mode 100644
index 0000000..9509cb8
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,9 @@
+- [ ] Is it mergeable?
+- [ ] `make test` Did it pass the tests?
+- [ ] `make clean diff-cover` If it introduces new functionality, is it tested?
+- [ ] `make format diff_pylint_report doc` Is it well formatted?
+- [ ] For substantial changes or changes to the command-line interface, is it
+ documented in `CHANGELOG.md`? See [keepachangelog](http://keepachangelog.com/)
+ for more details.
+- [ ] Was a spellchecker run on the source code and documentation after
+ changes were made?
diff --git a/.travis.yml b/.travis.yml
index bde74d3..58d4d6e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,8 +3,7 @@ python: 2.7
env:
- TOX_ENV=py27
- - TOX_ENV=py33
- - TOX_ENV=py34
+ - TOX_ENV=py35
install:
- pip install tox
diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 0000000..f485b6c
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,15 @@
+# Change Log
+All notable changes to the screed project will be documented in this file.
+See [keepachangelog](http://keepachangelog.com/) for more info.
+
+The screed Python and command-line APIs adhere to
+[Semantic Versioning](http://semver.org/).
+
+## [Unreleased]
+
+## [1.0.0] - 2017-03-29
+### Added
+- screed CLI, with database creation and conversion commands.
+- screed.make_db, a simplified way of creating DB using the Python API.
+- Unify record writing in the `write_fastx` function.
+- Update tests & constrain behavior for screed Records.
diff --git a/Makefile b/Makefile
index 7f5ee90..4be75c4 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@
PYSOURCES=$(wildcard screed/*.py)
TESTSOURCES=$(wildcard screed/tests/*.py)
SOURCES=$(PYSOURCES) setup.py
-DEVPKGS=pep8==1.5.7 diff_cover autopep8 pylint coverage nose sphinx
+DEVPKGS=pep8==1.5.7 diff_cover autopep8 pylint coverage pytest pytest-cov sphinx
VERSION=$(shell git describe --tags --dirty | sed s/v//)
all:
@@ -15,6 +15,7 @@ all:
install-dependencies:
pip install --upgrade $(DEVPKGS) || pip2 install --upgrade $(DEVPKGS)
+ pip install --upgrade --requirement doc/requirements.txt
install: FORCE
./setup.py build install
@@ -29,12 +30,13 @@ dist/screed-$(VERSION).tar.gz: $(SOURCES)
clean: FORCE
./setup.py clean --all || true
- rm coverage-debug || true
- rm -Rf .coverage || true
- rm -Rf doc/_build || true
+ rm -rf build/
+ rm -rf coverage-debug .coverage coverage.xml
+ rm -rf doc/_build
+ rm -rf .eggs/ *.egg-info/ .cache/ __pycache__/ *.pyc */*.pyc */*/*.pyc
pep8: $(PYSOURCES) $(TESTSOURCES)
- pep8 --exclude=_version.py setup.py screed/ || true
+ pep8 --exclude=_version.py setup.py screed/
pep8_report.txt: $(PYSOURCES) $(TESTSOURCES)
pep8 --exclude=_version.py setup.py screed/ > pep8_report.txt || true
@@ -62,8 +64,7 @@ diff_pylint_report: pylint_report.txt
diff-quality --violations=pylint pylint_report.txt
.coverage: $(PYSOURCES) $(TESTSOURCES)
- ./setup.py nosetests --with-coverage --cover-package screed \
- --attr '!known_failing' 2>&1 > .coverage.out
+ ./setup.py test --addopts="--cov"
coverage.xml: .coverage
coverage xml --omit 'screed/tests/*'
@@ -83,8 +84,8 @@ diff-cover: coverage.xml
diff-cover.html: coverage.xml
diff-cover coverage.xml --html-report diff-cover.html
-nosetests.xml: FORCE
- ./setup.py nosetests --with-xunit --attr '!known_failing'
+tests.xml: FORCE
+ ./setup.py test --addopts "--junitxml=$@"
doxygen: doc/doxygen/html/index.html
@@ -109,7 +110,7 @@ doc/doxygen/html/index.html: ${CPPSOURCES} ${PYSOURCES}
test: FORCE
./setup.py develop
- ./setup.py nosetests --attr '!known_failing'
+ ./setup.py test
sloccount.sc: ${PYSOURCES} Makefile
sloccount --duplicates --wide --details screed setup.py Makefile \
diff --git a/README.md b/README.md
index 97f4863..9780d26 100644
--- a/README.md
+++ b/README.md
@@ -8,4 +8,4 @@ See http://readthedocs.org/docs/screed/en/latest/ for docs.
Issues are tracked at https://github.com/dib-lab/khmer/issues.
-[](http://ci.ged.msu.edu/job/screed/)
+[](https://travis-ci.org/dib-lab/screed)
diff --git a/TODO b/TODO
deleted file mode 100644
index 65de8cc..0000000
--- a/TODO
+++ /dev/null
@@ -1,16 +0,0 @@
-Pragma statements in sqlite? http://www2.sqlite.org/pragma.html
- - synchronous - enable for writing, maybe not important for reading
- - locking_mode - can't do for reading, writing yes
- - disable all locking for reading maybe
- - cache_size
-
-fix the conversion so can be achieved with text -> text instead of
-db -> text
-
-==== 3/14/2010
-
-PEP-8 noncompliance, e.g. screedDB should be ScreedDB, although
-I kind of agree that screedDB is prettier.
-
-accuracy => quality
-get pairs via a standard api, with/without enforcement
diff --git a/benchmarks/faGen.py b/benchmarks/faGen.py
index 982f3c2..9f3a2dd 100755
--- a/benchmarks/faGen.py
+++ b/benchmarks/faGen.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
+# Copyright (c) 2016, The Regents of the University of California.
import sys, os
import random
diff --git a/benchmarks/fqGen.py b/benchmarks/fqGen.py
index e327a6a..fd51499 100755
--- a/benchmarks/fqGen.py
+++ b/benchmarks/fqGen.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
+# Copyright (c) 2016, The Regents of the University of California.
import sys, os
import random
diff --git a/benchmarks/fqToFaConvert.py b/benchmarks/fqToFaConvert.py
index 0983514..afe0dc7 100755
--- a/benchmarks/fqToFaConvert.py
+++ b/benchmarks/fqToFaConvert.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
+# Copyright (c) 2016, The Regents of the University of California.
import sys
import os
diff --git a/benchmarks/screedCreateTimeit.py b/benchmarks/screedCreateTimeit.py
index 933ef23..f194bb4 100755
--- a/benchmarks/screedCreateTimeit.py
+++ b/benchmarks/screedCreateTimeit.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
+# Copyright (c) 2016, The Regents of the University of California.
import sys
import timeit
diff --git a/benchmarks/screedTimeit.py b/benchmarks/screedTimeit.py
index aee3493..9ef6b9a 100755
--- a/benchmarks/screedTimeit.py
+++ b/benchmarks/screedTimeit.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
+# Copyright (c) 2016, The Regents of the University of California.
import timeit
import sys
diff --git a/benchmarks/screedTimeit1M.py b/benchmarks/screedTimeit1M.py
index f5f70e1..b4ed3fd 100755
--- a/benchmarks/screedTimeit1M.py
+++ b/benchmarks/screedTimeit1M.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
+# Copyright (c) 2016, The Regents of the University of California.
import timeit
import sys
diff --git a/bigtests/__init__.py b/bigtests/__init__.py
index c623fe9..ec94783 100755
--- a/bigtests/__init__.py
+++ b/bigtests/__init__.py
@@ -1,13 +1,18 @@
-import sys, os, gc
-import urllib, tarfile
+# Copyright (c) 2016, The Regents of the University of California.
+
+import gc
+import os
+import sys
+import tarfile
+import urllib
thisdir = os.path.dirname(__file__)
libdir = os.path.abspath(os.path.join(thisdir, '..', 'screed'))
sys.path.insert(0, libdir)
-from screed import read_fastq_sequences
-from screed import read_fasta_sequences
-from screed import ScreedDB
+from screed import read_fastq_sequences # nopep8
+from screed import read_fasta_sequences # nopep8
+from screed import ScreedDB # nopep8
tests22 = os.path.join(thisdir, 's_2_2_sequence.fastq')
tests31 = os.path.join(thisdir, 's_3_1_sequence.fastq')
@@ -18,6 +23,7 @@ mus = os.path.join(thisdir, 'Mus_musculus.NCBIM37.50.dna_rm.chromosome.9.fa')
xeno = os.path.join(thisdir, 'Xenopus_tropicalis.JGI4.1.50.dna.toplevel.fa')
sorex = os.path.join(thisdir, 'Sorex_araneus.COMMON_SHREW1.53.dna.toplevel.fa')
+
def getfile(f):
"""
Downloads and extracts the given file
@@ -32,8 +38,8 @@ def getfile(f):
try:
up = urllib.urlopen(base_url)
except IOError:
- raise IOError("Error downloading testfiles, are you connected to " +\
- "the internet?")
+ raise IOError("Error downloading testfiles, are you connected to "
+ "the internet?")
fp.write(up.read())
fp.close()
@@ -43,15 +49,18 @@ def getfile(f):
os.unlink(filename)
return
+
def setup():
# Create databases
endings = ['_screed']
- filenames = [(tests22, 'fastq'), (tests31, 'fastq'), (tests42, 'fastq'),
- (pongo, 'fasta'), (tri, 'fasta'), (mus, 'fasta'), (xeno, 'fasta'),
- (sorex, 'fasta')]
+ filenames = [
+ (tests22, 'fastq'), (tests31, 'fastq'), (tests42, 'fastq'),
+ (pongo, 'fasta'), (tri, 'fasta'), (mus, 'fasta'), (xeno, 'fasta'),
+ (sorex, 'fasta')
+ ]
for f in filenames:
fname = f[0]
- if not os.path.isfile(fname): # Download files if necessary
+ if not os.path.isfile(fname): # Download files if necessary
getfile(f)
parser = None
if f[1] == 'fasta':
@@ -62,9 +71,10 @@ def setup():
for end in endings:
if not os.path.isfile(fname + end):
created = False
- if created == False:
+ if not created:
parser(fname)
+
class Test_s22_fastq:
"""
Test screed methods on the s22 fastq file
@@ -101,7 +111,7 @@ class Test_s22_fastq:
for k in self.db:
assert k in self.db
- assert not 'FOO' in self.db
+ assert 'FOO' not in self.db
def test_get(self):
for k in self.db:
@@ -111,7 +121,7 @@ class Test_s22_fastq:
record = self.db[k]
assert record.name == k
- assert self.db.get('FOO') == None
+ assert self.db.get('FOO') is None
try:
self.db['FOO']
assert False, "the previous line should raise a KeyError"
@@ -170,7 +180,7 @@ class Test_s22_fastq:
'id': 0,
'annotations': '',
'quality': 'AA7AAA3+AAAAAA.AAA.;7;AA;;;;*;<1;<<<',
- 'name' : 'HWI-EAS_4_PE-FC20GCB:2:1:492:573/2',
+ 'name': 'HWI-EAS_4_PE-FC20GCB:2:1:492:573/2',
'sequence': 'ACAGCAAAATTGTGATTGAGGATGAAGAACTGCTGT'}
testcases['HWI-EAS_4_PE-FC20GCB:2:162:131:826/2'] = {
@@ -183,7 +193,7 @@ class Test_s22_fastq:
testcases['HWI-EAS_4_PE-FC20GCB:2:330:88:628/2'] = {
'id': 3790455,
'annotations': '',
- 'quality' : 'AA;AA??A5A;;+AA?AAAA;AA;9AA.AA?????9',
+ 'quality': 'AA;AA??A5A;;+AA?AAAA;AA;9AA.AA?????9',
'name': 'HWI-EAS_4_PE-FC20GCB:2:330:88:628/2',
'sequence': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAA'}
@@ -204,6 +214,7 @@ class Test_s22_fastq:
for case in testcases:
assert testcases[case] == self.db[case]
+
class Test_s31_fastq:
"""
Test screed methods on the s31 fastq file
@@ -240,7 +251,7 @@ class Test_s31_fastq:
for k in self.db:
assert k in self.db
- assert not 'FOO' in self.db
+ assert 'FOO' not in self.db
def test_get(self):
for k in self.db:
@@ -250,7 +261,7 @@ class Test_s31_fastq:
record = self.db[k]
assert record.name == k
- assert self.db.get('FOO') == None
+ assert self.db.get('FOO') is None
try:
self.db['FOO']
assert False, "the previous line should raise a KeyError"
@@ -309,7 +320,7 @@ class Test_s31_fastq:
'id': 0,
'annotations': '',
'quality': 'CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC',
- 'name' : 'HWI-EAS_4_PE-FC20GCB:3:1:71:840/1',
+ 'name': 'HWI-EAS_4_PE-FC20GCB:3:1:71:840/1',
'sequence': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'}
testcases['HWI-EAS_4_PE-FC20GCB:3:330:957:433/1'] = {
@@ -322,7 +333,7 @@ class Test_s31_fastq:
testcases['HWI-EAS_4_PE-FC20GCB:3:166:443:410/1'] = {
'id': 2219847,
'annotations': '',
- 'quality' : 'AAAAAAAAAAAAAAAAAAAAAAAA6<@AA959???%',
+ 'quality': 'AAAAAAAAAAAAAAAAAAAAAAAA6<@AA959???%',
'name': 'HWI-EAS_4_PE-FC20GCB:3:166:443:410/1',
'sequence': 'TGGCATTCGCACACATCATGATGGTGCTGACCGTAA'}
@@ -343,6 +354,7 @@ class Test_s31_fastq:
for case in testcases:
assert testcases[case] == self.db[case]
+
class Test_s42_fastq:
"""
Test screed methods on the s42 fastq file
@@ -364,7 +376,6 @@ class Test_s42_fastq:
nameRcrd = self.db[rcrd.name]
assert rcrd == nameRcrd
-
def test_dict_stuff(self):
"""
Tests some dictionary methods on the database
@@ -380,7 +391,7 @@ class Test_s42_fastq:
for k in self.db:
assert k in self.db
- assert not 'FOO' in self.db
+ assert 'FOO' not in self.db
def test_get(self):
for k in self.db:
@@ -390,7 +401,7 @@ class Test_s42_fastq:
record = self.db[k]
assert record.name == k
- assert self.db.get('FOO') == None
+ assert self.db.get('FOO') is None
try:
self.db['FOO']
assert False, "the previous line should raise a KeyError"
@@ -449,9 +460,9 @@ class Test_s42_fastq:
'id': 0,
'annotations': '',
'quality': 'AAAAAAAA:4>>AAA:44>>->-&4;8+8826;66.',
- 'name' : 'HWI-EAS_4_PE-FC20GCB:4:1:257:604/2',
+ 'name': 'HWI-EAS_4_PE-FC20GCB:4:1:257:604/2',
'sequence': 'TGTGGATAGTCGCCCGTGATGGCGTCGAAGTTCCGG'}
-
+
testcases['HWI-EAS_4_PE-FC20GCB:4:330:96:902/2'] = {
'id': 4148632,
'annotations': '',
@@ -462,7 +473,7 @@ class Test_s42_fastq:
testcases['HWI-EAS_4_PE-FC20GCB:4:166:158:532/2'] = {
'id': 2074316,
'annotations': '',
- 'quality' : 'AAAAAAA?A?AAAAAAA?A>A?A?AAAAAA?.<?-?',
+ 'quality': 'AAAAAAA?A?AAAAAAA?A>A?A?AAAAAA?.<?-?',
'name': 'HWI-EAS_4_PE-FC20GCB:4:166:158:532/2',
'sequence': 'ATCGCCAATGCCCAGGCCTGGTTCTCTTTAACCTAT'}
@@ -520,7 +531,7 @@ class Test_po_fasta:
for k in self.db:
assert k in self.db
- assert not 'FOO' in self.db
+ assert 'FOO' not in self.db
def test_get(self):
for k in self.db:
@@ -530,7 +541,7 @@ class Test_po_fasta:
record = self.db[k]
assert record.name == k
- assert self.db.get('FOO') == None
+ assert self.db.get('FOO') is None
try:
self.db['FOO']
assert False, "the previous line should raise a KeyError"
@@ -587,46 +598,48 @@ class Test_po_fasta:
testcases = {}
testcases['GENSCAN00000032971'] = {
'id': 0,
- 'description': 'cdna:Genscan chromosome:PPYG2:6_qbl_hap2_random' \
+ 'description': 'cdna:Genscan chromosome:PPYG2:6_qbl_hap2_random'
':95622:98297:1',
- 'name' : 'GENSCAN00000032971',
- 'sequence': 'ATGGCGCCCCGAACCCTCCTCCTGCTGCTCTCGGCGGCCCTGGCCCCGAC' \
+ 'name': 'GENSCAN00000032971',
+ 'sequence': 'ATGGCGCCCCGAACCCTCCTCCTGCTGCTCTCGGCGGCCCTGGCCCCGAC'
'CGAGACCTGG'}
testcases['GENSCAN00000042282'] = {
'id': 53997,
- 'description': 'cdna:Genscan chromosome:PPYG2:1:229892060:22989' \
+ 'description': 'cdna:Genscan chromosome:PPYG2:1:229892060:22989'
'2800:1',
'name': 'GENSCAN00000042282',
- 'sequence': 'ATGATGCCATTGCAAGGACCCTCTGCAGGGCCTCAGTCCCGAGGATGGCA' \
+ 'sequence': 'ATGATGCCATTGCAAGGACCCTCTGCAGGGCCTCAGTCCCGAGGATGGCA'
'CACAGCCTTC'}
testcases['GENSCAN00000051311'] = {
'id': 30780,
- 'description' : 'cdna:Genscan chromosome:PPYG2:10:132962172:132' \
+ 'description': 'cdna:Genscan chromosome:PPYG2:10:132962172:132'
'962871:1',
'name': 'GENSCAN00000051311',
- 'sequence': 'ATGACCCAGCCACCTACCAGGCCGCTCTGCAGACCCCCCACGGGAGCAGC' \
+ 'sequence': 'ATGACCCAGCCACCTACCAGGCCGCTCTGCAGACCCCCCACGGGAGCAGC'
'CTCTGCCCCC'}
testcases['GENSCAN00000006030'] = {
'id': 1469,
- 'description': 'cdna:Genscan chromosome:PPYG2:14_random:1765749' \
+ 'description': 'cdna:Genscan chromosome:PPYG2:14_random:1765749'
':1766075:-1',
'name': 'GENSCAN00000006030',
- 'sequence': 'ATGTGTGGCAACAAGGGCATTTCTGCCTTCCCTGAATCAGACCACCTTTT' \
+ 'sequence': 'ATGTGTGGCAACAAGGGCATTTCTGCCTTCCCTGAATCAGACCACCTTTT'
'CACATGGGTA'}
testcases['GENSCAN00000048263'] = {
'id': 43029,
- 'description': 'cdna:Genscan chromosome:PPYG2:6:100388173:10048' \
+ 'description': 'cdna:Genscan chromosome:PPYG2:6:100388173:10048'
'5454:-1',
'name': 'GENSCAN00000048263',
- 'sequence': 'ATGTGTCCCTTTGAATATGCCGGAGAACAACAGTTGCCATGGATGTGTTC' \
+ 'sequence': 'ATGTGTCCCTTTGAATATGCCGGAGAACAACAGTTGCCATGGATGTGTTC'
'TGGGGAGCCC'}
for case in testcases:
assert testcases[case]['name'] == self.db[case]['name']
- assert testcases[case]['description'] == self.db[case]\
- ['description']
- assert str(self.db[case]['sequence']).startswith(testcases[case]\
- ['sequence'])
+ assert testcases[case]['description'] == \
+ self.db[case]['description']
+ assert str(self.db[case]['sequence']).startswith(
+ testcases[case]['sequence']
+ )
+
class Test_mus_fasta:
"""
@@ -664,7 +677,7 @@ class Test_mus_fasta:
for k in self.db:
assert k in self.db
- assert not 'FOO' in self.db
+ assert 'FOO' not in self.db
def test_get(self):
for k in self.db:
@@ -674,7 +687,7 @@ class Test_mus_fasta:
record = self.db[k]
assert record.name == k
- assert self.db.get('FOO') == None
+ assert self.db.get('FOO') is None
try:
self.db['FOO']
assert False, "the previous line should raise a KeyError"
@@ -730,19 +743,21 @@ class Test_mus_fasta:
"""
testcases = {}
testcases['9'] = {
- 'id': 0,
- 'description': 'dna_rm:chromosome chromosome:NCBIM37:9:1:124076' \
- '172:1',
- 'name' : '9',
- 'sequence': 'NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN' \
- 'NNNNNNNNNN'}
+ 'id': 0,
+ 'description': 'dna_rm:chromosome chromosome:NCBIM37:9:1:124076'
+ '172:1',
+ 'name': '9',
+ 'sequence': 'NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN'
+ 'NNNNNNNNNN'
+ }
for case in testcases:
assert testcases[case]['name'] == self.db[case]['name']
- assert testcases[case]['description'] == self.db[case]\
- ['description']
- assert str(self.db[case]['sequence']).startswith(testcases[case]\
- ['sequence'])
+ assert testcases[case]['description'] == \
+ self.db[case]['description']
+ assert str(self.db[case]['sequence']).startswith(
+ testcases[case]['sequence']
+ )
class Test_tri_fasta:
@@ -781,7 +796,7 @@ class Test_tri_fasta:
for k in self.db:
assert k in self.db
- assert not 'FOO' in self.db
+ assert 'FOO' not in self.db
def test_get(self):
for k in self.db:
@@ -791,7 +806,7 @@ class Test_tri_fasta:
record = self.db[k]
assert record.name == k
- assert self.db.get('FOO') == None
+ assert self.db.get('FOO') is None
try:
self.db['FOO']
assert False, "the previous line should raise a KeyError"
@@ -849,7 +864,7 @@ class Test_tri_fasta:
testcases['singleUn_100'] = {
'id': 0,
'description': '',
- 'name' : 'singleUn_100',
+ 'name': 'singleUn_100',
'sequence': 'TTTAAACACGTGTCCGCGCCATTTTTTTATTTATTTACCGATCAAGTGCA'}
testcases['singleUn_9'] = {
'id': 2210,
@@ -858,7 +873,7 @@ class Test_tri_fasta:
'sequence': 'TTTAATTTTTTTACAACTCAAAATTTTGAGTAGTGTTTTAAATAGTACAC'}
testcases['ChLG6'] = {
'id': 2016,
- 'description' : '',
+ 'description': '',
'name': 'ChLG6',
'sequence': 'CAAAAAAATTCATAACTCAAAAACTAAAAGTCGTAGAGCAATGCGGTTTG'}
testcases['singleUn_286'] = {
@@ -874,10 +889,11 @@ class Test_tri_fasta:
for case in testcases:
assert testcases[case]['name'] == self.db[case]['name']
- assert testcases[case]['description'] == self.db[case]\
- ['description']
- assert str(self.db[case]['sequence']).startswith(testcases[case]\
- ['sequence'])
+ assert testcases[case]['description'] == \
+ self.db[case]['description']
+ assert str(self.db[case]['sequence']).startswith(
+ testcases[case]['sequence']
+ )
class Test_xeno_fasta:
@@ -916,7 +932,7 @@ class Test_xeno_fasta:
for k in self.db:
assert k in self.db
- assert not 'FOO' in self.db
+ assert 'FOO' not in self.db
def test_get(self):
for k in self.db:
@@ -926,7 +942,7 @@ class Test_xeno_fasta:
record = self.db[k]
assert record.name == k
- assert self.db.get('FOO') == None
+ assert self.db.get('FOO') is None
try:
self.db['FOO']
assert False, "the previous line should raise a KeyError"
@@ -983,46 +999,48 @@ class Test_xeno_fasta:
testcases = {}
testcases['scaffold_20095'] = {
'id': 0,
- 'description': 'dna:scaffold scaffold:JGI4.1:scaffold_20095:1:2'\
- '001:1',
- 'name' : 'scaffold_20095',
- 'sequence': 'GATGAGATCACCTTTCATGCTTTTTGTATCCCTATTATCTAGAGACAACAA'\
- 'ATCAGTTGC'}
+ 'description': 'dna:scaffold scaffold:JGI4.1:scaffold_20095:1:2'
+ '001:1',
+ 'name': 'scaffold_20095',
+ 'sequence': 'GATGAGATCACCTTTCATGCTTTTTGTATCCCTATTATCTAGAGACAACAA'
+ 'ATCAGTTGC'}
testcases['scaffold_1'] = {
'id': 19500,
- 'description': 'dna:scaffold scaffold:JGI4.1:scaffold_1:1:781781'\
- '4:1',
+ 'description': 'dna:scaffold scaffold:JGI4.1:scaffold_1:1:781781'
+ '4:1',
'name': 'scaffold_1',
- 'sequence': 'CCTCCCTTTTTGGCTGTCTTTTCACTGTATCATAGCCTGGCGTGAACCAAG'\
- 'CCTCAAAAA'}
+ 'sequence': 'CCTCCCTTTTTGGCTGTCTTTTCACTGTATCATAGCCTGGCGTGAACCAAG'
+ 'CCTCAAAAA'}
testcases['scaffold_271'] = {
'id': 19230,
- 'description' : 'dna:scaffold scaffold:JGI4.1:scaffold_271:1:156'\
- '7461:1',
+ 'description': 'dna:scaffold scaffold:JGI4.1:scaffold_271:1:156'
+ '7461:1',
'name': 'scaffold_271',
- 'sequence': 'CGATTTTTGCGGAAAAACGCGAGTTTTTGGTAGCCATTCCGAAAGTTGCGA'\
- 'TTTTTTGTA'}
+ 'sequence': 'CGATTTTTGCGGAAAAACGCGAGTTTTTGGTAGCCATTCCGAAAGTTGCGA'
+ 'TTTTTTGTA'}
testcases['scaffold_19901'] = {
'id': 329,
- 'description': 'dna:scaffold scaffold:JGI4.1:scaffold_19901:1:22'\
- '56:1',
+ 'description': 'dna:scaffold scaffold:JGI4.1:scaffold_19901:1:22'
+ '56:1',
'name': 'scaffold_19901',
- 'sequence': 'ATACCGCAAAGGTTTCTTTCTTCTCAGTGCTCCATGCTGCCTCTCTTGTTT'\
- 'TGCCTCCCT'}
+ 'sequence': 'ATACCGCAAAGGTTTCTTTCTTCTCAGTGCTCCATGCTGCCTCTCTTGTTT'
+ 'TGCCTCCCT'}
testcases['scaffold_95'] = {
'id': 19408,
- 'description': 'dna:scaffold scaffold:JGI4.1:scaffold_95:1:28996'\
- '70:1',
+ 'description': 'dna:scaffold scaffold:JGI4.1:scaffold_95:1:28996'
+ '70:1',
'name': 'scaffold_95',
- 'sequence': 'CCCTCCTGGTGATCCCACTTCAATCTCCCCATAGGCACACATCACTTCTAG'\
- 'CAGTTCACA'}
+ 'sequence': 'CCCTCCTGGTGATCCCACTTCAATCTCCCCATAGGCACACATCACTTCTAG'
+ 'CAGTTCACA'}
for case in testcases:
assert testcases[case]['name'] == self.db[case]['name']
- assert testcases[case]['description'] == self.db[case]\
- ['description']
- assert str(self.db[case]['sequence']).startswith(testcases[case]\
- ['sequence'])
+ assert testcases[case]['description'] == \
+ self.db[case]['description']
+ assert str(self.db[case]['sequence']).startswith(
+ testcases[case]['sequence']
+ )
+
class Test_sorex_fasta:
"""
@@ -1060,7 +1078,7 @@ class Test_sorex_fasta:
for k in self.db:
assert k in self.db
- assert not 'FOO' in self.db
+ assert 'FOO' not in self.db
def test_get(self):
for k in self.db:
@@ -1070,7 +1088,7 @@ class Test_sorex_fasta:
record = self.db[k]
assert record.name == k
- assert self.db.get('FOO') == None
+ assert self.db.get('FOO') is None
try:
self.db['FOO']
assert False, "the previous line should raise a KeyError"
@@ -1127,55 +1145,56 @@ class Test_sorex_fasta:
testcases = {}
testcases['scaffold_93039'] = {
'id': 0,
- 'description': 'dna:scaffold scaffold:COMMON_SHREW1:scaffold_93'\
- '039:1:203:1',
+ 'description': 'dna:scaffold scaffold:COMMON_SHREW1:scaffold_93'
+ '039:1:203:1',
'name': 'scaffold_93039',
- 'sequence': 'GCTGAGCCTTGTAGTTCTGCTCCCTTTGACTGACGGCCCACTATGGACCG'\
- 'GAAAAACTAC'}
-
+ 'sequence': 'GCTGAGCCTTGTAGTTCTGCTCCCTTTGACTGACGGCCCACTATGGACCG'
+ 'GAAAAACTAC'}
+
testcases['scaffold_107701'] = {
'id': 1,
- 'description': 'dna:scaffold scaffold:COMMON_SHREW1:scaffold_10'\
- '7701:1:203:1',
- 'name' : 'scaffold_107701',
- 'sequence': 'TAAACCCAAAATAAACATTCCCCAAATTATATTTCTTCCTTTCCTTCTGA'\
- 'ATAAAAGAAA'}
-
+ 'description': 'dna:scaffold scaffold:COMMON_SHREW1:scaffold_10'
+ '7701:1:203:1',
+ 'name': 'scaffold_107701',
+ 'sequence': 'TAAACCCAAAATAAACATTCCCCAAATTATATTTCTTCCTTTCCTTCTGA'
+ 'ATAAAAGAAA'}
+
testcases['GeneScaffold_6994'] = {
'id': 243135,
- 'description': 'dna:genescaffold genescaffold:COMMON_SHREW1:Gen'\
- 'eScaffold_6994:1:2349312:1',
+ 'description': 'dna:genescaffold genescaffold:COMMON_SHREW1:Gen'
+ 'eScaffold_6994:1:2349312:1',
'name': 'GeneScaffold_6994',
- 'sequence': 'TATTGAGAGAAGTGGGAACTTCTCTAGTGGTGGGGTATGGTGATGGAATG'\
- 'ATGTATGAAT'}
-
+ 'sequence': 'TATTGAGAGAAGTGGGAACTTCTCTAGTGGTGGGGTATGGTGATGGAATG'
+ 'ATGTATGAAT'}
+
testcases['scaffold_118324'] = {
'id': 13823,
- 'description': 'dna:scaffold scaffold:COMMON_SHREW1:scaffold_11'\
- '8324:1:884:1',
+ 'description': 'dna:scaffold scaffold:COMMON_SHREW1:scaffold_11'
+ '8324:1:884:1',
'name': 'scaffold_118324',
- 'sequence': 'CAGCCCCCTGCAACAAATTTTATACTCTAGAAACAGTTTAATGGCTGTTG'\
- 'GAATATTTCC'}
-
+ 'sequence': 'CAGCCCCCTGCAACAAATTTTATACTCTAGAAACAGTTTAATGGCTGTTG'
+ 'GAATATTTCC'}
+
testcases['scaffold_92895'] = {
'id': 14573,
- 'description': 'dna:scaffold scaffold:COMMON_SHREW1:scaffold_92'\
- '895:1:890:1',
+ 'description': 'dna:scaffold scaffold:COMMON_SHREW1:scaffold_92'
+ '895:1:890:1',
'name': 'scaffold_92895',
- 'sequence': 'GGGAAGCTTGCAAGGCTGTCCCATGTGGGCAGGAAGCTCTCAGTAGCTTG'\
- 'CCAGTTTCTC'}
-
+ 'sequence': 'GGGAAGCTTGCAAGGCTGTCCCATGTGGGCAGGAAGCTCTCAGTAGCTTG'
+ 'CCAGTTTCTC'}
+
testcases['scaffold_62271'] = {
'id': 37101,
- 'description': 'dna:scaffold scaffold:COMMON_SHREW1:scaffold_62'\
- '271:1:1064:1',
+ 'description': 'dna:scaffold scaffold:COMMON_SHREW1:scaffold_62'
+ '271:1:1064:1',
'name': 'scaffold_62271',
- 'sequence': 'AGAGTATCTCCCCCACATGGCAGAGCCTGGCAAGCTACCCATGGCGTATT'\
- 'CAATATGCCA'}
+ 'sequence': 'AGAGTATCTCCCCCACATGGCAGAGCCTGGCAAGCTACCCATGGCGTATT'
+ 'CAATATGCCA'}
for case in testcases:
assert testcases[case]['name'] == self.db[case]['name']
assert testcases[case]['description'] == \
- self.db[case]['description']
- assert str(self.db[case]['sequence']).startswith(testcases[case]\
- ['sequence'])
+ self.db[case]['description']
+ assert str(self.db[case]['sequence']).startswith(
+ testcases[case]['sequence']
+ )
diff --git a/doc/LICENSE.rst b/doc/LICENSE.rst
index 2660056..696ac57 100644
--- a/doc/LICENSE.rst
+++ b/doc/LICENSE.rst
@@ -2,13 +2,29 @@
License
=======
-Copyright (c) 2008-2015, Michigan State University
+Copyright (c) 2008, Michigan State University.
+Copyright (c) 2015, The Regents of the University of California.
All rights reserved.
-Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
- * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
- * Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ * Neither the name of the author nor the names of its contributors may be
+ used to endorse or promote products derived from this software without
+ specific prior written permission.
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFI [...]
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/doc/conf.py b/doc/conf.py
index e893f5a..7cf910f 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -3,7 +3,8 @@
# screed documentation build configuration file, created by
# sphinx-quickstart on Wed Jun 6 16:32:37 2012.
#
-# This file is execfile()d with the current directory set to its containing dir.
+# This file is execfile()d with the current directory set to its containing
+# dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
@@ -11,30 +12,37 @@
# All configuration values have a default; values that are commented out
# serve to show the default.
-import sys, os
+import sys
+import os
+
+from recommonmark.parser import CommonMarkParser
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
+# sys.path.insert(0, os.path.abspath('.'))
-# -- General configuration -----------------------------------------------------
+# -- General configuration ----------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
+# needs_sphinx = '1.0'
-# Add any Sphinx extension module names here, as strings. They can be extensions
-# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = []
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
+source_parsers = {
+ '.md': CommonMarkParser,
+}
+
# The suffix of source filenames.
-source_suffix = '.rst'
+source_suffix = ['.rst', '.md']
# The encoding of source files.
-#source_encoding = 'utf-8-sig'
+# source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
@@ -50,61 +58,51 @@ copyright = u'2012-2015, Michigan State University'
# The full version, including alpha/beta/rc tags.
-sys.path.insert(0, '.')
-sys.path.insert(0, '../') # to get the versioneer module
-
-import imp
-fp, pathname, description = imp.find_module('versioneer')
-versioneer = imp.load_module('versioneer', fp, pathname, description)
-del imp
-
-versioneer.VCS = 'git'
-versioneer.versionfile_source = '../screed/_version.py'
-versioneer.versionfile_build = '../screed/_version.py'
-versioneer.tag_prefix = 'v' # tags are like v1.2.0
-versioneer.parentdir_prefix = '..'
-
-release = versioneer.get_version()
+from screed._version import get_versions
+release = get_versions()['version']
+del get_versions
# The short X.Y version.
-version = '.'.join(release.split('.')[:2])
+version = '.'.join(release.split('.')[:2])
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
-#language = None
+# language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
-#today = ''
+# today = ''
+
# Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+# today_fmt = '%B %d, %Y'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']
-# The reST default role (used for this markup: `text`) to use for all documents.
-#default_role = None
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+# default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
+# add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
-#add_module_names = True
+# add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
-#show_authors = False
+# show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
+# modindex_common_prefix = []
-# -- Options for HTML output ---------------------------------------------------
+# -- Options for HTML output --------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
@@ -113,26 +111,26 @@ html_theme = 'default'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
-#html_theme_options = {}
+# html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
+# html_theme_path = []
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
-#html_title = None
+# html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
-#html_short_title = None
+# html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
-#html_logo = None
+# html_logo = None
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
-#html_favicon = None
+# html_favicon = None
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
@@ -142,71 +140,72 @@ html_style = 'labibi.css'
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
+# html_last_updated_fmt = '%b %d, %Y'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
-#html_use_smartypants = True
+# html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+# html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
-#html_additional_pages = {}
+# html_additional_pages = {}
# If false, no module index is generated.
-#html_domain_indices = True
+# html_domain_indices = True
# If false, no index is generated.
-#html_use_index = True
+# html_use_index = True
# If true, the index is split into individual pages for each letter.
-#html_split_index = False
+# html_split_index = False
# If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
+# html_show_sourcelink = True
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
+# html_show_sphinx = True
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
+# html_show_copyright = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
-#html_use_opensearch = ''
+# html_use_opensearch = ''
# This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
+# html_file_suffix = None
# Output file base name for HTML help builder.
htmlhelp_basename = 'screeddoc'
html_context = {
- "google_analytics_id" : 'UA-51731094-1',
- "disqus_shortname" : 'screed-docs',
- # "github_base_account" : 'dib-lab',
- "github_project" : 'screed',
+ "google_analytics_id": 'UA-51731094-1',
+ "disqus_shortname": 'screed-docs',
+ # "github_base_account": 'dib-lab',
+ "github_project": 'screed',
}
-# -- Options for LaTeX output --------------------------------------------------
+# -- Options for LaTeX output -------------------------------------------------
latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
+ # The paper size ('letterpaper' or 'a4paper').
+ # 'papersize': 'letterpaper',
-# The font size ('10pt', '11pt' or '12pt').
-#'pointsize': '10pt',
+ # The font size ('10pt', '11pt' or '12pt').
+ # 'pointsize': '10pt',
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
+ # Additional stuff for the LaTeX preamble.
+ # 'preamble': '',
}
# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title, author, documentclass [howto/manual]).
+# (source start file, target name, title, author, documentclass
+# [howto/manual]).
latex_documents = [
('index', 'screed.tex', u'screed Documentation',
u'Alex Nolley and Titus Brown', 'manual'),
@@ -214,26 +213,26 @@ latex_documents = [
# The name of an image file (relative to this directory) to place at the top of
# the title page.
-#latex_logo = None
+# latex_logo = None
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
-#latex_use_parts = False
+# latex_use_parts = False
# If true, show page references after internal links.
-#latex_show_pagerefs = False
+# latex_show_pagerefs = False
# If true, show URL addresses after external links.
-#latex_show_urls = False
+# latex_show_urls = False
# Documents to append as an appendix to all manuals.
-#latex_appendices = []
+# latex_appendices = []
# If false, no module index is generated.
-#latex_domain_indices = True
+# latex_domain_indices = True
-# -- Options for manual page output --------------------------------------------
+# -- Options for manual page output -------------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
@@ -243,25 +242,24 @@ man_pages = [
]
# If true, show URL addresses after external links.
-#man_show_urls = False
+# man_show_urls = False
-# -- Options for Texinfo output ------------------------------------------------
+# -- Options for Texinfo output -----------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
- ('index', 'screed', u'screed Documentation',
- u'Alex Nolley and Titus Brown', 'screed', 'One line description of project.',
- 'Miscellaneous'),
+ ('index', 'screed', u'screed Documentation', u'Alex Nolley and Titus Brown',
+ 'screed', 'One line description of project.', 'Miscellaneous'),
]
# Documents to append as an appendix to all manuals.
-#texinfo_appendices = []
+# texinfo_appendices = []
# If false, no module index is generated.
-#texinfo_domain_indices = True
+# texinfo_domain_indices = True
# How to display URL addresses: 'footnote', 'no', or 'inline'.
-#texinfo_show_urls = 'footnote'
+# texinfo_show_urls = 'footnote'
diff --git a/doc/dev/coding-guidelines-and-review.rst b/doc/dev/coding-guidelines-and-review.rst
index 2da7992..c99c004 100644
--- a/doc/dev/coding-guidelines-and-review.rst
+++ b/doc/dev/coding-guidelines-and-review.rst
@@ -52,12 +52,12 @@ Checklist
Copy and paste the following into a pull request comment when it is
ready for review::
- - [ ] Is it mergable?
- - [ ] Did it pass the tests?
- - [ ] If it introduces new functionality, is it tested?
- - [ ] Is it well formatted? Look at `make pep8`, `make diff_pylint_report`,
- and `make doc` output. Use `make format` and manual fixing as needed.
- - [ ] Is it documented in the ChangeLog?
+ - [ ] Is it mergeable?
+ - [ ] `make test` Did it pass the tests?
+ - [ ] `make clean diff-cover` If it introduces new functionality, is it tested?
+ - [ ] `make format diff_pylint_report doc` Is it well formatted?
+ - [ ] Is it documented in the `ChangeLog`?
+ http://en.wikipedia.org/wiki/Changelog#Format
- [ ] Was a spellchecker run on the source code and documentation after
changes were made?
diff --git a/doc/dev/index.rst b/doc/dev/index.rst
index 98ddcca..8bc67e5 100644
--- a/doc/dev/index.rst
+++ b/doc/dev/index.rst
@@ -1,5 +1,3 @@
-.. vim set filetype=rst
-
The screed developer documentation
==================================
@@ -16,5 +14,6 @@ Contents:
.. toctree::
:maxdepth: 1
+ parsers
coding-guidelines-and-review
release-checklist
diff --git a/doc/dev/parsers.rst b/doc/dev/parsers.rst
new file mode 100644
index 0000000..41f3c13
--- /dev/null
+++ b/doc/dev/parsers.rst
@@ -0,0 +1,167 @@
+Writing Custom Sequence Parsers
+===============================
+
+screed is built to be adaptable to new kinds of file sequence formats.
+Included with screed are parsers for handling FASTA and FASTQ sequence
+file types, though if you need screed to work with a new format, all
+you need to do is write a new parser.
+
+Field Roles
+-----------
+
+Each field in a screed database is assigned a role. These roles
+describe what kind of information is stored in their field. Right now
+there are only 4 different roles in a screed database: the text role,
+the sliceable role, the indexed key role and the primary key role. All
+roles are defined in the file: screed/DBConstants.py
+
+The text role (DBConstants._STANDARD_TEXT) is the role most fields in
+a database will have. This role tells screed that the associated field
+is storing standard textual data. Nothing special.
+
+The sliceable role (DBConstants._SLICEABLE_TEXT) is a role that can be
+assigned to long sequence fields. screed's default FASTA parser
+defines the 'sequence' field with the sliceable role. When screed
+retrieves a field that has the sliceable role, it builds a special
+data structure that supports slicing into the text.
+
+The indexed key role (DBConstants._INDEXED_TEXT_KEY) is associated
+with exactly one of the fields in a screed database. In screed's FASTA
+and FASTQ parsers, this role is fulfilled by the 'name' field. This
+field is required because it is the field screed tells sqlite to index
+when creating the database and it is the field used for name look-ups
+when querying a screed database.
+
+The primary key role (DBConstants._PRIMARY_KEY_ROLE) is a role
+automatically associated with the 'id' field in each database. This
+field is always created with each screed database and always holds
+this role. You as a user of screed won't need to worry about this one.
+
+General Parsing Function Format
+-------------------------------
+
+create_db is the function central to the creation of screed
+databases. This function accepts a file path, a tuple of field names
+and roles, and an iterator function. The file path describes where the
+screed database should go, the tuple contains the names of fields and
+their associated roles and the iterator function yields records in a
+dictionary format.
+
+This sub-section describes general steps for preparing and using
+screed with a custom sequence parser. Though they don't have to be,
+future sequence parsers should be located in the seqparse.py file for
+convenience. These steps will be described in the context of working
+from the Python shell.
+
+First import the create_db function::
+
+ >>> from screed import create_db
+
+The create_db class handles the formatting of screed databases and
+provides a simple interface for storing sequence data.
+
+Next the database fields and roles must be specified. The fields tell
+screed the names and order of the data fields inside each record. For instance,
+lets say our new sequence has types 'name', 'bar', and 'baz', all text. The
+tuple will be::
+
+ >>> fields = (('name', DBConstants._INDEXED_TEXT_KEY),
+ ('bar', DBConstants._STANDARD_TEXT),
+ ('baz', DBConstants._STANDARD_TEXT))
+
+Notice how 'name' is given the indexed key role and bar and baz are
+given text roles? If, for instance, you know 'baz' fields can be very long
+and you want to be able to retrieve slices of them, you could specify
+fields as::
+
+ >>> fields = (('name', DBConstants._INDEXED_TEXT_KEY),
+ ('bar', DBConstants._STANDARD_TEXT),
+ ('baz', DBConstants._SLICEABLE_TEXT))
+
+All screed databases come with an 'id' field, which is a sequential
+numbering order starting at 0 for the first record, 1 for the second, and
+so on. The names and number of the other fields are arbitrary with one
+restriction: one and only one of the fields must fulfill the indexed key role.
+
+Next, you need to setup an iterator function that will return records in
+a dictionary format. Have a look at the 'fastq_iter', 'fasta_iter', or
+'hava_iter' functions in the screed/fastq.py, screed/fasta.py, and
+screed/hava.py files, respectively for examples on how to write one of these.
+If you don't know what an iterator function is, the documentation on the
+Python website gives a good description:
+http://docs.python.org/library/stdtypes.html#iterator-types.
+
+Once the iterator function is written, it needs to be instantiated. In the
+context of the built-in parsing functions, this means opening a file and
+passing the file handle to the iterator function::
+
+ >>> seqfile = open('path_to_seq_file', 'rb')
+ >>> iter_instance = myiter(seqfile)
+
+Assuming that your iterator function is called 'myiter', this sets up an
+instance of it ready to use with create_db.
+
+Now the screed database is created with one command::
+
+ >>> create_db('path_to_screed_db', fields, iter_instance)
+
+If you want the screed database saved at 'path_to_screed_db'. If instead you
+want the screed database created in the same directory and with a
+similar file name as the sequence file, its OK to do this::
+
+ >>> create_db('path_to_seq_file', fields, iter_instance)
+
+create_db will just append '_screed' to the end of the file name and make
+a screed database at that file path so the original file won't be
+overwritten.
+
+When you're done the sequence file should be closed::
+
+ >>> seqfile.close()
+
+Using the Built-in Sequence Iterator Functions
+----------------------------------------------
+
+This section shows how to use the 'fastq_iter' and 'fasta_iter' functions
+for returning records from a sequence file.
+
+These functions both take a file handle as the only argument and then return
+a dictionary for each record in the file containing names of fields and
+associated data. These functions are primarily used in conjunction with
+the db_create() function, but they can be useful by themselves.
+
+First, import the necessary module and open a text file containing sequences.
+For this example, the 'fastq_iter' function will be used::
+
+ >>> import screed.fastq
+ >>> seqfile = open('path_to_seqfile', 'rb')
+
+Now, the 'fastq_iter' can be instantiated and iterated over::
+
+ >>> fq_instance = screed.fastq(seqfile)
+ >>> for record in fq_instance:
+ ... print record.name
+
+That will print the name of every sequence in the file. If instead you want
+to accumulate the sequences::
+
+ >>> sequences = []
+ >>> for record in fq_instance:
+ ... sequences.append(record.sequence)
+
+These iterators are the core of screed's sequence modularity. If there is
+a new sequence format you want screed to work with, all it needs is its
+own iterator.
+
+Error checking in parsing methods
+---------------------------------
+
+The existing FASTA/FASTQ parsing functions contain some error
+checking, such as making sure the file can be opened and checking
+correct data is being read. Though screed doesn't enforce this, it is
+strongly recommended to include error checking code in your parser. To
+remain non-specific to one file sequence type or another, the
+underlying screed library can't contain error checking code of this
+kind. If errors are not detected by the parsing function, they will be
+silently included into the database being built and could cause
+problems much later when trying to read from the database.
diff --git a/doc/dev/release-checklist.rst b/doc/dev/release-checklist.rst
index 4ea4317..b12cb55 100644
--- a/doc/dev/release-checklist.rst
+++ b/doc/dev/release-checklist.rst
@@ -25,12 +25,13 @@ Getting Started
#. Install/update versioneer::
pip install versioneer
- versioneer-installer
+ versioneer install
- If there is a new version of versioneer::
+ If there is a new version of versioneer, follow the instruction to update
+ it and fix the configuration if needed::
git diff
- ./setup.py versioneer
+ versioneer install
git commit -p -m "new version of versioneer.py"
# or abandon the changes
git checkout -- versioneer.py screed/_version.py screed/__init.py \
@@ -45,7 +46,7 @@ Getting Started
#. Review the issue list for any existing bugs that won't be fixed in the
release and ensure they're documented in ``doc/known-issues.txt``
-#. Verify that the build is clean: http://ci.ged.msu.edu/job/screed/
+#. Verify that the build is clean: https://travis-ci.org/dib-lab/screed
#. Set the new version number and release candidate::
@@ -68,27 +69,26 @@ Getting Started
# first we test the tag
cd testenv1
source bin/activate
- pip install nose
git clone --depth 1 --branch v${new_version}-${rc} \
https://github.com/dib-lab/screed.git
cd screed
+ make install-dependencies
make install
- nosetests screed --attr '!known_failing'
make test
- python -c 'import screed; print screed.__version__' # double-check version number
+ python -c 'import screed; print(screed.__version__)' # double-check version number
# Test via pip
cd ../../testenv2
source bin/activate
- pip install nose
pip install -e \
git+https://github.com/dib-lab/screed.git@v${new_version}-${rc}#egg=screed
cd src/screed
make dist
make install
- nosetests screed --attr '!known_failing'
- python -c 'import screed; print screed.__version__'
+ pip install pytest
+ pytest --pyargs screed -m 'not known_failing'
+ python -c 'import screed; print(screed.__version__)'
cp dist/screed*tar.gz ../../../testenv3
# test if the dist made in testenv2 is complete enough to build another
@@ -96,10 +96,10 @@ Getting Started
cd ../../../testenv3
source bin/activate
- pip install nose
+ pip install pytest
pip install screed*tar.gz
- nosetests screed --attr '!known_failing'
- python -c 'import screed; print screed.__version__'
+ pytest --pyargs screed -m 'not known_failing'
+ python -c 'import screed; print(screed.__version__)'
tar xzf screed*tar.gz
cd screed*
make dist
@@ -119,13 +119,13 @@ Getting Started
cd ../../testenv4
source bin/activate
- pip install -U setuptools
- pip install nose
+ pip install -U setuptools pip
+ pip install pytest
pip install -i https://testpypi.python.org/pypi --pre --no-clean screed
- nosetests screed --attr '!known_failing'
- python -c 'import screed; print screed.__version__'
+ pytest --pyargs screed -m 'not known_failing'
+ python -c 'import screed; print(screed.__version__)'
cd build/screed
- ./setup.py nosetests --attr '!known_failing'
+ ./setup.py test
#. Do any final testing (acceptance tests, etc.) Note that the acceptance tests
for screed are to run the khmer automated tests with the new version of
diff --git a/doc/index.rst b/doc/index.rst
index 0e6057e..734428a 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -6,7 +6,8 @@
screed - short read sequence utils
==================================
-:Copyright: 2008-2012 Michigan State University
+:Copyright: 2008, Michigan State University.
+:Copyright: 2015, The Regents of the University of California.
:Authors: Alex Nolley, C. Titus Brown
:Contact: ctb at msu.edu
:License: BSD
@@ -18,18 +19,17 @@ Contents:
screed
example
-
+
dev/index
release-notes/index
user/known-issues
-
+
CODE_OF_CONDUCT
LICENSE
-
+
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
-
diff --git a/doc/release-notes/RELEASE-1.0.md b/doc/release-notes/RELEASE-1.0.md
new file mode 100644
index 0000000..1ab67ad
--- /dev/null
+++ b/doc/release-notes/RELEASE-1.0.md
@@ -0,0 +1,52 @@
+# Release 1.0
+
+We are pleased to announce the release of screed 1.0. Screed is a
+biological sequence parsing and storage/retrieval library for DNA and
+protein sequences. It's designed to be lightweight and easy to use
+from Python.
+
+This version is the first with API compatibility guarantees,
+following the [semantic versioning guidelines][0].
+Most changes are internal or API clarifications,
+but there is a new shell command for screed functions and an unified function
+for writing FAST{A,Q} records.
+
+Documentation is available at http://screed.readthedocs.org/en/v1.0
+
+## New items of note:
+
+- New shell commands for common screed operations:
+ - `db` for database creation (`screed db <filename>`)
+ - dumping FAST{A,Q} records from a db
+ (`screed dump_fasta <db> <output>`
+ and `screed dump_fastq <db> <output>`). #55 @luizirber
+- Remove `\*_Writer` classes and unify record writing in the `write_fastx`
+ function. #53 @standage
+- We now use pytest as a test runner,
+ codecov for code coverage,
+ and a simplified changelog format. #50 #49 #59 @luizirber @standage
+
+## Other bugs fixed/issues closed:
+
+- Fix reverse complement problems for Python 2.7. #47 @ctb
+- Fix operator comparison. #48 @luizirber
+- Update tests & constrain behavior for screed Records. #54 @ctb
+- Allow sqlite3 import to fail. #56 @ctb
+- Cleanup user docs and code. #62 #57 @standage
+- Simplify use of 'open' internally. #65 @ctb
+
+## Known Issues
+
+These are all pre-existing
+
+ - Screed does not support gzip file streaming. This is an issue with Python 2.x and will likely *not* be fixed in future releases. This is being tracked in dib-lab/khmer#700
+
+ - Screed is overly tolerant of spaces in fast{a,q} which is against spec. This is being tracked in dib-lab/khmer#108
+
+## Contributors
+
+ at luizirber \*@standage @ctb \*@betatim
+
+\* Indicates new contributors
+
+[0]: http://semver.org/
diff --git a/doc/release-notes/index.rst b/doc/release-notes/index.rst
index 42f33eb..185dd6e 100644
--- a/doc/release-notes/index.rst
+++ b/doc/release-notes/index.rst
@@ -8,5 +8,7 @@ Contents:
.. toctree::
:maxdepth: 1
- RELEASE-0.5
+ RELEASE-1.0
+ RELEASE-0.9
RELEASE-0.8
+ RELEASE-0.5
diff --git a/doc/requirements.txt b/doc/requirements.txt
new file mode 100644
index 0000000..f5f889d
--- /dev/null
+++ b/doc/requirements.txt
@@ -0,0 +1 @@
+recommonmark==0.4.0
diff --git a/doc/screed.rst b/doc/screed.rst
index a4a4c3f..503e689 100755
--- a/doc/screed.rst
+++ b/doc/screed.rst
@@ -1,33 +1,24 @@
-===================
-Basic Documentation
-===================
-
-.. contents:
+===========
+User Manual
+===========
-Notes on this document
-======================
-This is the default documentation for screed. Some doctests are included
-in the file 'example.txt'. The examples in this file are meant for humans
-only: they will not work in doctests.
+.. note::
-Introduction
-============
+ Some doctests are included in :doc:`example`. The examples in this
+ document are meant for human consumption only. They will not work in
+ doctests!
-screed parses FASTA and FASTQ files, generates databases, and lets you
-query these databases. Values such as sequence name, sequence
-description, sequence quality, and the sequence itself can be
-retrieved from these databases.
+screed parses FASTA and FASTQ files, generates databases, and lets you query
+these databases. Values such as sequence name, sequence description, sequence
+quality, and the sequence itself can be retrieved from these databases.
-Getting Going
-=============
+Installation
+============
The following software packages are required to run screed:
-* Python 2.4 or newer
-* nose (for testing)
-
-Installing
------------
+* Python 2 (2.7) or Python 3 (3.3 or newer)
+* pytest (only required to running tests)
Use pip to download, and install Screed and its dependencies::
@@ -35,123 +26,120 @@ Use pip to download, and install Screed and its dependencies::
To run the optional tests type::
- nosetests screed --attr '!known_failing'
-
-Quick-Start
-===========
+ python -m screed.tests
-Reading FASTA/FASTQ files
--------------------------
+Command-line Quick Start
+========================
-At the Python prompt, type::
+Creating a database
+-------------------
- >>> import screed
- >>> with screed.open(filename) as seqfile:
- >>> for read in seqfile:
- ... print read.name, read.sequence
+.. code::
-Here, 'filename' can be a FASTA or FASTQ file, and can be
-uncompressed, gzipped, or bz2-zipped.
+ $ screed db <fasta/fastq file>
-Creating a database from the API
---------------------------------
+Dumping a database to a file
+----------------------------
-From a Python prompt type::
+.. code::
- >>> import screed
- >>> screed.read_fasta_sequences('screed/tests/test.fa')
+ $ screed dump_fasta <screed database file> <fasta output>
+ $ screed dump_fastq <screed database file> <fastq output>
-That command just parsed the FASTA file 'screed/tests/test.fa' into a
-screed-database named 'screed/tests/test.fa_screed'. The screed database
-is independent from the text file it was derived from, so moving, renaming
-or deleting the 'screed/tests/test.fa' file will not affect
-screed's operation. To create a screed database from a FASTQ file the
-syntax is similar::
+If no output file is provided, sequences are written to the terminal (stdout) by
+default.
- >>> screed.read_fastq_sequences('screed/tests/test.fastq')
+Python Quick Start
+==================
-Creating a database from a script
----------------------------------
+Reading FASTA/FASTQ files
+-------------------------
-If the screed module is in your PYTHONPATH, you can create a screed db from
-a FASTQ file at the shell::
+ >>> import screed
+ >>> with screed.open(filename) as seqfile:
+ >>> for read in seqfile:
+ ... print(read.name, read.sequence)
- $ python -m screed.fqdbm <fastq file>
+Here, :code:`filename` can be a FASTA or FASTQ file, and can be uncompressed,
+gzip-compressed, or bzip2-compressed. screed natively supports FASTA and FASTQ
+databases creation. If your sequences are in a different format see the
+developer documentation on :doc:`dev/parsers`.
-Similarly, to create a screed db from a fasta file::
+Creating a database
+-------------------
- $ python -m screed.fadbm <fasta file>
+ >>> import screed
+ >>> screed.make_db('screed/tests/test-data/test.fa')
-where <fast* file> is the path to a sequence file.
+This loads a FASTA file :code:`screed/tests/test-data/test.fa` into a screed database
+named :code:`screed/tests/test-data/test.fa_screed`. A couple of things to note:
-screed natively supports FASTA and FASTQ database creation. If you
-have a new sequence you want screed to work with, see the section
-below on Writing Custom Sequence Parsers.
+* The screed database is independent of the text file from which it was derived,
+ so moving, renaming or deleting :code:`screed/tests/test-data/test.fa` will not affect
+ the newly created database.
+* The :code:`make_db` function inferred the file type as FASTA automatically.
+ The :code:`read_fasta_sequences()` and :code:`read_fastq_sequences()`
+ functions are available if you'd prefer to be explicit.
-Reading databases
-=================
+ >>> screed.read_fasta_sequences('screed/tests/test-data/test.fasta')
+ >>> screed.read_fastq_sequences('screed/tests/test-data/test.fastq')
-The class ScreedDB is used to read screed databases, regardless of
-what file format they were derived from (FASTA/FASTQ/hava/etc.). One
-reader to rule them all!
+Opening a database
+------------------
-Opening
--------
+The class :code:`ScreedDB` is used to read screed databases, regardless of what
+file format they were derived from (FASTA/FASTQ/hava/etc.). One reader to rule
+them all!
-In the Python environment, import the ScreedDB class and load some
-databases::
+From the Python prompt, import the ScreedDB class and load some databases::
>>> from screed import ScreedDB
- >>> fadb = ScreedDB('screed/tests/test.fa')
- >>> fqdb = ScreedDB('screed/tests/test.fastq')
+ >>> fadb = ScreedDB('screed/tests/test-data/test.fa')
+ >>> fqdb = ScreedDB('screed/tests/test-data/test.fastq')
-Notice how you didn't need to write the '_screed' at the end of the
-file names? screed automatically adds that to the file name if you
-didn't.
+Notice how you didn't need to write the '_screed' at the end of the file names?
+screed automatically adds that to the file name if you didn't.
-Dictionary Interface
---------------------
+Database dictionary interface
+-----------------------------
-Since screed emulates a read-only dictionary interface, any methods
-that don't modify a dictionary are supported::
+Since screed emulates a read-only dictionary interface, any methods that don't
+modify a dictionary are supported::
>>> fadb.keys()
>>> fqdb.keys()
-Each record in the database contains 'fields' such as name and
-sequence information. If the database was derived from a FASTQ file,
-quality and optional annotation strings are included. Conversely,
-FASTA-derived databases have a description field.
+Each record in the database contains 'fields' such as name and sequence
+information. If the database was derived from a FASTQ file, quality and optional
+annotation strings are included. Conversely, FASTA-derived databases have a
+description field.
To retrieve the names of records in the database::
>>> names = fadb.keys()
-Length of the databases are easily found::
+The size of the databases (number of sequence records) is easily found::
- >>> print len(fadb)
+ >>> len(fadb)
22
- >>> print len(fqdb)
+ >>> len(fqdb)
125
-Retrieving Records
-------------------
+Retrieving records from a database
+----------------------------------
-A record is the standard container unit in screed. Each has 'fields'
-that vary slightly depending on what kind of file the database was
-derived from. For instance, a FASTQ-derived screed database has an
-id, a name, a quality score and a sequence. A FASTA-derived screed
-database has an id, name, description and a sequence.
+A record is the standard container unit in screed. Each has *fields* that vary
+slightly depending on what kind of file the database was derived from. For
+instance, a FASTQ-derived screed database has an id, a name, a quality score and
+a sequence. A FASTA-derived screed database has an id, name, description and a
+sequence.
-Retrieving whole records::
+Retrieving entire records::
- >>> records = []
- >>> for record in fadb.itervalues():
- ... records.append(record)
+ >>> records = [r for r in fadb.itervalues()]
-What is returned is a dictionary of fields. The names of fields
-are keys into this dictionary with the actual information as values.
-For example::
+Each record is a dictionary of fields. The names of fields are keys into this
+dictionary with the actual information as values. For example::
>>> record = fadb[fadb.keys()[0]]
>>> index = record['id']
@@ -159,23 +147,23 @@ For example::
>>> description = record['description']
>>> sequence = record['sequence']
-What this does is retrieve the first record object in the screed database,
-then retrieve the index, name, description and sequence from the record
-object using standard dictionary key -> value pairs.
+What this does is retrieve the first record object in the screed database, then
+retrieve the index, name, description and sequence from the record object using
+standard dictionary key -> value pairs.
-Retrieving Partial Sequences (slicing)
+Retrieving partial sequences (slicing)
--------------------------------------
-screed supports the concept of retrieving a 'slice' or a subset of a
-sequence string. The motivation is speed: if you have a database entry
-with a very long sequence string but only want a small portion of the
-string, it is faster to retrieve only the portion than to retrieve the
-entire string and then perform standard Python string slicing.
+screed supports the concept of retrieving a *slice* or a subset of a sequence
+string. The motivation is speed: if you have a database entry with a very long
+sequence string but only want a small portion of the string, it is faster to
+retrieve only the portion than to retrieve the entire string and then perform
+standard Python string slicing.
-By default, screed's FASTA database creator sets up the 'sequence'
-column to support slicing. For example, if you have an entry with name
-'someSeq' which has a 10K long sequence, and you want a slice of the
-sequence spanning positions 4000 to 4080::
+By default, screed's FASTA database creator sets up the :code:`sequence` column
+to support slicing. For example, if you have an entry with name :code:`someSeq`
+which has a 10K long sequence, and you want a slice of the sequence spanning
+positions 4000 to 4080::
>>> seq = db['someSeq'].sequence
>>> slice = seq[4000:4080]
@@ -185,245 +173,75 @@ This is much faster than say::
>>> seq = str(db['someSeq'].sequence)
>>> slice = seq[4000:4080]
-Because deep down, less information is being read off the disk. The
-str() method above causes the entire sequence to be retrieved as a
-string. Then Python slicing is done on the string 'seq' and the subset
-stored in 'slice'.
+Because deep down, less information is being read off the disk. The :code`str()`
+method above causes the entire sequence to be retrieved as a string. Then Python
+slicing is done on the string :code:`seq` and the subset stored in
+:code:`slice`.
-Retrieving Via Index
---------------------
+Retrieving records *via* index
+------------------------------
-Sometimes you don't care what the name of a sequence is; you're only
-interested in its position in the database. In these cases, retrieval
-via index is the method you'll want to use::
+Sometimes you don't care what the name of a sequence is; you're only interested
+in its position in the database. In these cases, retrieval via index is the
+method you'll want to use::
>>> record = fqdb.loadRecordByIndex(5)
-An index is like an offset into the database. The order records were
-kept in the FASTA or FASTQ file determines the index in their
-resulting screed database. The first record in a sequence file will
-have an index of 0, the second, an index of 1 and so on.
-
-Writing Custom Sequence Parsers
-===============================
-
-screed is built to be adaptable to new kinds of file sequence formats.
-Included with screed are parsers for handling FASTA and FASTQ sequence
-file types, though if you need screed to work with a new format, all
-you need to do is write a new parser.
-
-Field Roles
------------
-
-Each field in a screed database is assigned a role. These roles
-describe what kind of information is stored in their field. Right now
-there are only 4 different roles in a screed database: the text role,
-the sliceable role, the indexed key role and the primary key role. All
-roles are defined in the file: screed/DBConstants.py
-
-The text role (DBConstants._STANDARD_TEXT) is the role most fields in
-a database will have. This role tells screed that the associated field
-is storing standard textual data. Nothing special.
-
-The sliceable role (DBConstants._SLICEABLE_TEXT) is a role that can be
-assigned to long sequence fields. screed's default FASTA parser
-defines the 'sequence' field with the sliceable role. When screed
-retrieves a field that has the sliceable role, it builds a special
-data structure that supports slicing into the text.
-
-The indexed key role (DBConstants._INDEXED_TEXT_KEY) is associated
-with exactly one of the fields in a screed database. In screed's FASTA
-and FASTQ parsers, this role is fulfilled by the 'name' field. This
-field is required because it is the field screed tells sqlite to index
-when creating the database and it is the field used for name look-ups
-when querying a screed database.
-
-The primary key role (DBConstants._PRIMARY_KEY_ROLE) is a role
-automatically associated with the 'id' field in each database. This
-field is always created with each screed database and always holds
-this role. You as a user of screed won't need to worry about this one.
-
-General Parsing Function Format
--------------------------------
-
-create_db is the function central to the creation of screed
-databases. This function accepts a file path, a tuple of field names
-and roles, and an iterator function. The file path describes where the
-screed database should go, the tuple contains the names of fields and
-their associated roles and the iterator function yields records in a
-dictionary format.
-
-This sub-section describes general steps for preparing and using
-screed with a custom sequence parser. Though they don't have to be,
-future sequence parsers should be located in the seqparse.py file for
-convenience. These steps will be described in the context of working
-from the Python shell.
-
-First import the create_db function::
+An index is like an offset into the database. The order records were kept in the
+FASTA or FASTQ file determines the index in their resulting screed database. The
+first record in a sequence file will have an index of 0, the second, an index of
+1 and so on.
- >>> from screed import create_db
-
-The create_db class handles the formatting of screed databases and
-provides a simple interface for storing sequence data.
-
-Next the database fields and roles must be specified. The fields tell
-screed the names and order of the data fields inside each record. For instance,
-lets say our new sequence has types 'name', 'bar', and 'baz', all text. The
-tuple will be::
-
- >>> fields = (('name', DBConstants._INDEXED_TEXT_KEY),
- ('bar', DBConstants._STANDARD_TEXT),
- ('baz', DBConstants._STANDARD_TEXT))
-
-Notice how 'name' is given the indexed key role and bar and baz are
-given text roles? If, for instance, you know 'baz' fields can be very long
-and you want to be able to retrieve slices of them, you could specify
-fields as::
-
- >>> fields = (('name', DBConstants._INDEXED_TEXT_KEY),
- ('bar', DBConstants._STANDARD_TEXT),
- ('baz', DBConstants._SLICEABLE_TEXT))
-
-All screed databases come with an 'id' field, which is a sequential
-numbering order starting at 0 for the first record, 1 for the second, and
-so on. The names and number of the other fields are arbitrary with one
-restriction: one and only one of the fields must fulfill the indexed key role.
-
-Next, you need to setup an iterator function that will return records in
-a dictionary format. Have a look at the 'fastq_iter', 'fasta_iter', or
-'hava_iter' functions in the screed/fastq.py, screed/fasta.py, and
-screed/hava.py files, respectively for examples on how to write one of these.
-If you don't know what an iterator function is, the documentation on the
-Python website gives a good description:
-http://docs.python.org/library/stdtypes.html#iterator-types.
-
-Once the iterator function is written, it needs to be instantiated. In the
-context of the built-in parsing functions, this means opening a file and
-passing the file handle to the iterator function::
-
- >>> seqfile = open('path_to_seq_file', 'rb')
- >>> iter_instance = myiter(seqfile)
-
-Assuming that your iterator function is called 'myiter', this sets up an
-instance of it ready to use with create_db.
-
-Now the screed database is created with one command::
-
- >>> create_db('path_to_screed_db', fields, iter_instance)
-
-If you want the screed database saved at 'path_to_screed_db'. If instead you
-want the screed database created in the same directory and with a
-similar file name as the sequence file, its OK to do this::
-
- >>> create_db('path_to_seq_file', fields, iter_instance)
-
-create_db will just append '_screed' to the end of the file name and make
-a screed database at that file path so the original file won't be
-overwritten.
-
-When you're done the sequence file should be closed::
-
- >>> seqfile.close()
-
-Using the Built-in Sequence Iterator Functions
-----------------------------------------------
-
-This section shows how to use the 'fastq_iter' and 'fasta_iter' functions
-for returning records from a sequence file.
-
-These functions both take a file handle as the only argument and then return
-a dictionary for each record in the file containing names of fields and
-associated data. These functions are primarily used in conjunction with
-the db_create() function, but they can be useful by themselves.
-
-First, import the necessary module and open a text file containing sequences.
-For this example, the 'fastq_iter' function will be used::
-
- >>> import screed.fastq
- >>> seqfile = open('path_to_seqfile', 'rb')
-
-Now, the 'fastq_iter' can be instantiated and iterated over::
-
- >>> fq_instance = screed.fastq(seqfile)
- >>> for record in fq_instance:
- ... print record.name
-
-That will print the name of every sequence in the file. If instead you want
-to accumulate the sequences::
-
- >>> sequences = []
- >>> for record in fq_instance:
- ... sequences.append(record.sequence)
-
-These iterators are the core of screed's sequence modularity. If there is
-a new sequence format you want screed to work with, all it needs is its
-own iterator.
-
-Error checking in parsing methods
----------------------------------
-
-The existing FASTA/FASTQ parsing functions contain some error
-checking, such as making sure the file can be opened and checking
-correct data is being read. Though screed doesn't enforce this, it is
-strongly recommended to include error checking code in your parser. To
-remain non-specific to one file sequence type or another, the
-underlying screed library can't contain error checking code of this
-kind. If errors are not detected by the parsing function, they will be
-silently included into the database being built and could cause
-problems much later when trying to read from the database.
-
-File formats as understood by screed
+File Formats As Understood By Screed
====================================
-While the screed database remains non-specific to file formats, the
-included FASTA and FASTQ parsers expect specific formats. These
-parsers attempt to handle the most common attributes of sequence
-files, though they can not support all features.
+While the screed database remains non-specific to file formats, the included
+FASTA and FASTQ parsers expect specific formats. These parsers attempt to handle
+the most common attributes of sequence files, though they can not support all
+features.
FASTQ
-----
-The FASTQ parsing function is read_fastq_sequences() and is located in
+The FASTQ parsing function is :code:`read_fastq_sequences()` and is located in
the screed module.
-The first line in a record must begin with '@' and is followed by a
-record identifier (a name). An optional annotations string may be
-included after a space on the same line.
+The first line in a record must begin with '@' and is followed by a record
+identifier (a name). An optional annotations string may be included after a
+space on the same line.
-The second line begins the sequence line(s) which may be line wrapped.
-screed defines no limit on the length of sequence lines and no length
-on how many sequence lines a record may contain.
+The second line begins the sequence line(s) which may be line wrapped. screed
+defines no limit on the length of sequence lines and no length on how many
+sequence lines a record may contain.
-After the sequence line(s) comes a '+' character on a new line. Some
-FASTQ formats require the first line to be repeated after the '+'
-character, but since this adds no new information to the record,
-read_fastq_sequences() will ignore this if it is included.
+After the sequence line(s) comes a '+' character on a new line. Some FASTQ
+formats require the first line to be repeated after the '+' character, but since
+this adds no new information to the record, :code:`read_fastq_sequences()` will
+ignore this if it is included.
-The quality line(s) is last. Like the sequence line(s) this may
-be line wrapped. read_fastq_sequences() will raise an exception if the
-quality and sequence strings are of unequal length. screed performs
-no checking for valid quality scores.
+The quality line(s) is last. Like the sequence line(s) this may be line wrapped.
+:code:`read_fastq_sequences()` will raise an exception if the quality and
+sequence strings are of unequal length. screed performs no checking for valid
+quality scores.
FASTA
-----
-The FASTA parsing function is read_fasta_sequences() and is also
-located in the screed module.
+The FASTA parsing function is read_fasta_sequences() and is also located in the
+screed module.
-The first line in a record must begin with '>' and is followed with
-the sequence's name and an optional description. If the description is
-included, it is separated from the name with a space. Note that though
-the FASTA format doesn't require named records, screed does. Without a
-unique name, screed can't look up sequences by name.
+The first line in a record must begin with '>' and is followed with the
+sequence's name and an optional description. If the description is included, it
+is separated from the name with a space. Note that though the FASTA format
+doesn't require named records, screed does. Without a unique name, screed can't
+look up sequences by name.
The second line begins the line(s) of sequence. Like the FASTQ parser,
-read_fasta_sequences() allows any number of lines of any length.
+:code:`read_fasta_sequences()` allows any number of lines of any length.
FASTA <-> FASTQ Conversion
==========================
- at CTB this doesn't work?
-
As an extra nicety, screed can convert FASTA files to FASTQ and back again.
FASTA to FASTQ
@@ -435,7 +253,7 @@ first argument and a path to the desired FASTQ file as the second
argument. There is also a shell interface if the screed module is in
your PYTHONPATH::
- $ python -m screed.dump_to_fastq <path to fasta db> <converted fastq file>
+ $ python -m screed dump_fastq <path to fasta db> [ <converted fastq file> ]
The FASTA name attribute is directly dumped from the file. The
sequence attribute is also dumped pretty much directly, but is line
@@ -464,7 +282,7 @@ first argument and a path to the desired FASTA file as the second
argument. Like the ToFastq function before, there is a shell interface
to ToFasta if the screed module is in your PYTHONPATH::
- $ python -m screed.dump_to_fasta <path to fastq db> <converted fasta file>
+ $ python -m screed dump_fasta <path to fastq db> [ <converted fasta file> ]
As above, the name and sequence attributes are directly dumped from
the FASTQ database to the FASTA file with the sequence line wrapping
@@ -472,10 +290,3 @@ to 80 characters.
If it exists, the FASTQ annotation tag is stored as the FASTA description tag.
As there is no equivalent in FASTA, the FASTQ quality score is ignored.
-
-..
- Local Variables:
- mode: rst
- mode: outline-minor
- End:
-
diff --git a/get_version.py b/get_version.py
index d9ad6d9..849535f 100755
--- a/get_version.py
+++ b/get_version.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
+# Copyright (c) 2016, The Regents of the University of California.
""" Extracts the version of screed """
import sys
diff --git a/ChangeLog b/legacy/ChangeLog
similarity index 70%
rename from ChangeLog
rename to legacy/ChangeLog
index 12f1302..214f653 100644
--- a/ChangeLog
+++ b/legacy/ChangeLog
@@ -1,3 +1,55 @@
+2016-11-14 Daniel Standage <daniel.standage at gmail.com>
+
+ * screed/{__init__.py,fasta.py,fastq.py,openscreed.py}: Remove implementation
+ (and related references) of Writer classes.
+ * screed/tests/{test_open.py,test_open_cm.py}: Remove Writer tests.
+ * screed/{screedRecord.py,{tests/test_{fasta,fastq}.py}}: New write_fastx
+ implementation and associated tests.
+
+2016-11-14 Luiz Irber <khmer at luizirber.org>
+
+ * Makefile,doc/dev/release-checklist.rst,doc/screed.rst,jenkins-build.sh,
+ pytest.ini, setup.{cfg,py}, tox.ini, screed/tests/{__main__,havaGen,
+ screed_tst_utils,test_pygr_api, test_streaming}: Replace nose and adapt
+ for pytest.
+
+2016-10-13 Daniel Standage <daniel.standage at gmail.com>
+
+ * .travis.yml: Reduce the size of the CI build.
+
+2016-10-07 Luiz Irber <khmer at luizirber.org>
+
+ * screed/f{a,q}dbm.py: Fix import errors on Python 3.
+ * screed/tests/test_shell.py: check for return code and rewrite the test to
+ work more like the expected usage in the shell.
+
+2016-10-06 Luiz Irber <khmer at luizirber.org>
+
+ * tox.ini: Use codecov for coverage reports, add Python 3.5 to builds.
+ * .travis.yml: Activate Python 3.5 build.
+ * .github/{CONTRIBUTING,PULL_REQUEST_TEMPLATE}.md: Add GitHub templates.
+ * Makefile: Throw an error If there are pep8 warnings.
+ * doc/dev/coding-guidelinbes-and-review.rst: Update checklist
+ * screed/{dna,openscreed,tests/test_attriberror}.py: Fix pep8 warnings.
+
+2016-10-04 Luiz Irber <khmer at luizirber.org>
+
+ * screed/screedRecord.py: Implement comparison using total_ordering
+ decorator from functools.
+ * screed/tests/test_attriberror.py: Fix syntax errors for Python 3 and
+ remove tests for not implemented methods (they are implemented now).
+
+2016-06-10 Titus Brown <titus at idyll.org>
+
+ * screed/dna.py: Fix reverse complement calculation for Python 2.7
+
+2015-06-22 Jacob Fenton <bocajnotnef at gmail.com>
+
+ * screed/tests/test_attriberror.py: added tests to check screed db attribute
+ exception throwing
+ * screed/screedRecord.py: removed __cmp__ function, explicitly disallowed
+ all rich comparator functions that aren't == or !=
+
2015-06-10 Michael R. Crusoe <crusoe at ucdavis.edu>
* doc/user/known-issues.rst: removed two fixed issues
@@ -64,7 +116,7 @@
* doc/dev/release-checklist.txt: added "making final release" notes
* Makefile: copied over @mr-c's md-to-rst release notes conversion target
- * doc/dev/release-notes/RELEASE-0.8.txt: added rst version of release notes
+ * doc/dev/release-notes/RELEASE-0.8.txt: added rst version of release notes
for sphinx
* doc/dev/release-notes/index.txt: added rst version of 0.8 release notes to
toctree
@@ -107,7 +159,7 @@
2015-02-23 Michael R. Crusoe <mcrusoe at msu.edu>
* setup.py: work around versioneer bug:
- https://github.com/warner/python-versioneer/issues/52
+ https://github.com/warner/python-versioneer/issues/52
2014-12-07 Michael R. Crusoe <mcrusoe at msu.edu>
diff --git a/jenkins-build.sh b/legacy/jenkins-build.sh
similarity index 96%
rename from jenkins-build.sh
rename to legacy/jenkins-build.sh
index 9b0a6cf..84782af 100755
--- a/jenkins-build.sh
+++ b/legacy/jenkins-build.sh
@@ -12,7 +12,7 @@ virtualenv -p ${PYTHON_EXECUTABLE} .env
make install-dependencies > install_dependencies.out
make develop
make coverage.xml
-make nosetests.xml
+make tests.xml
if type doxygen >/dev/null 2>&1
then
make doxygen 2>&1 > doxygen.out
diff --git a/screed/DBConstants.py b/screed/DBConstants.py
index 8de0a1f..ba2c12e 100644
--- a/screed/DBConstants.py
+++ b/screed/DBConstants.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2008-2010, Michigan State University
+# Copyright (c) 2008, Michigan State University.
"""
Defines some constant strings identifications used in multiple
diff --git a/screed/__init__.py b/screed/__init__.py
index 2432977..98f4131 100755
--- a/screed/__init__.py
+++ b/screed/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2008-2010, Michigan State University
+# Copyright (c) 2008, Michigan State University.
"""
screed is a database tool useful for retrieving arbitrary kinds of sequence
@@ -24,11 +24,11 @@ ToFasta functions
from __future__ import absolute_import
-from screed.openscreed import ScreedDB, open_writer
-from screed.openscreed import open_reader as open
+from screed.openscreed import ScreedDB
+from screed.openscreed import Open as open
from screed.conversion import ToFastq
from screed.conversion import ToFasta
-from screed.createscreed import create_db
+from screed.createscreed import create_db, make_db
from screed.seqparse import read_fastq_sequences
from screed.seqparse import read_fasta_sequences
from screed.dna import rc
@@ -37,3 +37,7 @@ from screed.screedRecord import Record
from screed._version import get_versions
__version__ = get_versions()['version']
del get_versions
+
+from ._version import get_versions
+__version__ = get_versions()['version']
+del get_versions
diff --git a/screed/__main__.py b/screed/__main__.py
new file mode 100644
index 0000000..de71d12
--- /dev/null
+++ b/screed/__main__.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2016, The Regents of the University of California.
+
+from __future__ import absolute_import, print_function
+
+import argparse
+import sys
+
+from . import createscreed
+from . import dump_fasta
+from . import dump_fastq
+
+
+class ScreedCommands(object):
+
+ def __init__(self):
+ parser = argparse.ArgumentParser(
+ description="",
+ usage='''screed <command> [<args>]
+
+Available:
+
+ db <filename> Creates a screed database.
+ dump_fasta <db> <output> Convert a screed database to a FASTA file
+ dump_fastq <db> <output> Convert a screed database to a FASTQ file
+
+''')
+
+ commands = {
+ 'db': createscreed.main,
+ 'dump_fasta': dump_fasta.main,
+ 'dump_fastq': dump_fastq.main,
+ }
+
+ parser.add_argument('command')
+ args = parser.parse_args(sys.argv[1:2])
+ if args.command not in commands:
+ print('Unrecognized command')
+ parser.print_help()
+ sys.exit(1)
+
+ cmd = commands[args.command]
+ cmd(sys.argv[2:])
+
+
+def main():
+ ScreedCommands()
+ return 0
+
+if __name__ == "__main__":
+ main()
diff --git a/screed/_version.py b/screed/_version.py
index 574bc64..770a838 100644
--- a/screed/_version.py
+++ b/screed/_version.py
@@ -6,7 +6,9 @@
# that just contains the computed version number.
# This file is released into the public domain. Generated by
-# versioneer-0.14 (https://github.com/warner/python-versioneer)
+# versioneer-0.18 (https://github.com/warner/python-versioneer)
+
+"""Git implementation of _version.py."""
import errno
import os
@@ -14,23 +16,68 @@ import re
import subprocess
import sys
-# these strings will be replaced by git during git-archive
-git_refnames = " (HEAD -> master, tag: v0.9)"
-git_full = "1600a6c305deb0268d9b6066b728477b5a5a48b1"
-# these strings are filled in when 'setup.py versioneer' creates _version.py
-tag_prefix = "v"
-parentdir_prefix = "."
-versionfile_source = "screed/_version.py"
+def get_keywords():
+ """Get the keywords needed to look up the version information."""
+ # these strings will be replaced by git during git-archive.
+ # setup.py/versioneer.py will grep for the variable names, so they must
+ # each be defined on a line of their own. _version.py will just call
+ # get_keywords().
+ git_refnames = " (HEAD -> master, tag: v1.0)"
+ git_full = "34f34592c9984adcbadb3f665574cfb6bb60fa6e"
+ git_date = "2017-04-05 17:03:51 -0700"
+ keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
+ return keywords
+
+
+class VersioneerConfig:
+ """Container for Versioneer configuration parameters."""
+
+
+def get_config():
+ """Create, populate and return the VersioneerConfig() object."""
+ # these strings are filled in when 'setup.py versioneer' creates
+ # _version.py
+ cfg = VersioneerConfig()
+ cfg.VCS = "git"
+ cfg.style = "git-describe"
+ cfg.tag_prefix = "v"
+ cfg.parentdir_prefix = "'.'"
+ cfg.versionfile_source = "screed/_version.py"
+ cfg.verbose = False
+ return cfg
+
+class NotThisMethod(Exception):
+ """Exception raised if a method is not valid for the current scenario."""
-def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
+
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method): # decorator
+ """Decorator to mark a method as the handler for a particular VCS."""
+ def decorate(f):
+ """Store f in HANDLERS[vcs][method]."""
+ if vcs not in HANDLERS:
+ HANDLERS[vcs] = {}
+ HANDLERS[vcs][method] = f
+ return f
+ return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
+ env=None):
+ """Call the given command(s)."""
assert isinstance(commands, list)
p = None
for c in commands:
try:
+ dispcmd = str([c] + args)
# remember shell=False, so use git.cmd on windows, not just git
- p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
+ p = subprocess.Popen([c] + args, cwd=cwd, env=env,
+ stdout=subprocess.PIPE,
stderr=(subprocess.PIPE if hide_stderr
else None))
break
@@ -39,36 +86,52 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
if e.errno == errno.ENOENT:
continue
if verbose:
- print("unable to run %s" % args[0])
+ print("unable to run %s" % dispcmd)
print(e)
- return None
+ return None, None
else:
if verbose:
print("unable to find command, tried %s" % (commands,))
- return None
+ return None, None
stdout = p.communicate()[0].strip()
if sys.version_info[0] >= 3:
stdout = stdout.decode()
if p.returncode != 0:
if verbose:
- print("unable to run %s (error)" % args[0])
- return None
- return stdout
-
+ print("unable to run %s (error)" % dispcmd)
+ print("stdout was %s" % stdout)
+ return None, p.returncode
+ return stdout, p.returncode
+
+
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+ """Try to determine the version from the parent directory name.
+
+ Source tarballs conventionally unpack into a directory that includes both
+ the project name and a version string. We will also support searching up
+ two directory levels for an appropriately named parent directory
+ """
+ rootdirs = []
+
+ for i in range(3):
+ dirname = os.path.basename(root)
+ if dirname.startswith(parentdir_prefix):
+ return {"version": dirname[len(parentdir_prefix):],
+ "full-revisionid": None,
+ "dirty": False, "error": None, "date": None}
+ else:
+ rootdirs.append(root)
+ root = os.path.dirname(root) # up a level
-def versions_from_parentdir(parentdir_prefix, root, verbose=False):
- # Source tarballs conventionally unpack into a directory that includes
- # both the project name and a version string.
- dirname = os.path.basename(root)
- if not dirname.startswith(parentdir_prefix):
- if verbose:
- print("guessing rootdir is '%s', but '%s' doesn't start with "
- "prefix '%s'" % (root, dirname, parentdir_prefix))
- return None
- return {"version": dirname[len(parentdir_prefix):], "full": ""}
+ if verbose:
+ print("Tried directories %s but none started with prefix %s" %
+ (str(rootdirs), parentdir_prefix))
+ raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+ at register_vcs_handler("git", "get_keywords")
def git_get_keywords(versionfile_abs):
+ """Extract version information from the given file."""
# the code embedded in _version.py can just fetch the value of these
# keywords. When used from setup.py, we don't want to import _version.py,
# so we do it with a regexp instead. This function is not used from
@@ -85,20 +148,35 @@ def git_get_keywords(versionfile_abs):
mo = re.search(r'=\s*"(.*)"', line)
if mo:
keywords["full"] = mo.group(1)
+ if line.strip().startswith("git_date ="):
+ mo = re.search(r'=\s*"(.*)"', line)
+ if mo:
+ keywords["date"] = mo.group(1)
f.close()
except EnvironmentError:
pass
return keywords
-def git_versions_from_keywords(keywords, tag_prefix, verbose=False):
+ at register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+ """Get version information from git keywords."""
if not keywords:
- return {} # keyword-finding function failed to find keywords
+ raise NotThisMethod("no keywords at all, weird")
+ date = keywords.get("date")
+ if date is not None:
+ # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
+ # datestamp. However we prefer "%ci" (which expands to an "ISO-8601
+ # -like" string, which we must then edit to make compliant), because
+ # it's been around since git-1.5.3, and it's too difficult to
+ # discover which version we're using, or to work around using an
+ # older one.
+ date = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
refnames = keywords["refnames"].strip()
if refnames.startswith("$Format"):
if verbose:
print("keywords are unexpanded, not using")
- return {} # unexpanded, so not in an unpacked git-archive tarball
+ raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
refs = set([r.strip() for r in refnames.strip("()").split(",")])
# starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
# just "foo-1.0". If we see a "tag: " prefix, prefer those.
@@ -114,7 +192,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose=False):
# "stabilization", as well as "HEAD" and "master".
tags = set([r for r in refs if re.search(r'\d', r)])
if verbose:
- print("discarding '%s', no digits" % ",".join(refs-tags))
+ print("discarding '%s', no digits" % ",".join(refs - tags))
if verbose:
print("likely tags: %s" % ",".join(sorted(tags)))
for ref in sorted(tags):
@@ -124,116 +202,319 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose=False):
if verbose:
print("picking %s" % r)
return {"version": r,
- "full": keywords["full"].strip()}
+ "full-revisionid": keywords["full"].strip(),
+ "dirty": False, "error": None,
+ "date": date}
# no suitable tags, so version is "0+unknown", but full hex is still there
if verbose:
print("no suitable tags, using unknown + full revision id")
return {"version": "0+unknown",
- "full": keywords["full"].strip()}
+ "full-revisionid": keywords["full"].strip(),
+ "dirty": False, "error": "no suitable tags", "date": None}
+
+ at register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+ """Get version from 'git describe' in the root of the source tree.
-def git_parse_vcs_describe(git_describe, tag_prefix, verbose=False):
- # TAG-NUM-gHEX[-dirty] or HEX[-dirty] . TAG might have hyphens.
+ This only gets called if the git-archive 'subst' keywords were *not*
+ expanded, and _version.py hasn't already been rewritten with a short
+ version string, meaning we're inside a checked out source tree.
+ """
+ GITS = ["git"]
+ if sys.platform == "win32":
+ GITS = ["git.cmd", "git.exe"]
- # dirty
+ out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
+ hide_stderr=True)
+ if rc != 0:
+ if verbose:
+ print("Directory %s not under git control" % root)
+ raise NotThisMethod("'git rev-parse --git-dir' returned error")
+
+ # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+ # if there isn't one, this yields HEX[-dirty] (no NUM)
+ describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
+ "--always", "--long",
+ "--match", "%s*" % tag_prefix],
+ cwd=root)
+ # --long was added in git-1.5.5
+ if describe_out is None:
+ raise NotThisMethod("'git describe' failed")
+ describe_out = describe_out.strip()
+ full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+ if full_out is None:
+ raise NotThisMethod("'git rev-parse' failed")
+ full_out = full_out.strip()
+
+ pieces = {}
+ pieces["long"] = full_out
+ pieces["short"] = full_out[:7] # maybe improved later
+ pieces["error"] = None
+
+ # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+ # TAG might have hyphens.
+ git_describe = describe_out
+
+ # look for -dirty suffix
dirty = git_describe.endswith("-dirty")
+ pieces["dirty"] = dirty
if dirty:
git_describe = git_describe[:git_describe.rindex("-dirty")]
- dirty_suffix = ".dirty" if dirty else ""
# now we have TAG-NUM-gHEX or HEX
- if "-" not in git_describe: # just HEX
- return "0+untagged.g"+git_describe+dirty_suffix, dirty
+ if "-" in git_describe:
+ # TAG-NUM-gHEX
+ mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+ if not mo:
+ # unparseable. Maybe git-describe is misbehaving?
+ pieces["error"] = ("unable to parse git-describe output: '%s'"
+ % describe_out)
+ return pieces
+
+ # tag
+ full_tag = mo.group(1)
+ if not full_tag.startswith(tag_prefix):
+ if verbose:
+ fmt = "tag '%s' doesn't start with prefix '%s'"
+ print(fmt % (full_tag, tag_prefix))
+ pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
+ % (full_tag, tag_prefix))
+ return pieces
+ pieces["closest-tag"] = full_tag[len(tag_prefix):]
- # just TAG-NUM-gHEX
- mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
- if not mo:
- # unparseable. Maybe git-describe is misbehaving?
- return "0+unparseable"+dirty_suffix, dirty
+ # distance: number of commits since tag
+ pieces["distance"] = int(mo.group(2))
- # tag
- full_tag = mo.group(1)
- if not full_tag.startswith(tag_prefix):
- if verbose:
- fmt = "tag '%s' doesn't start with prefix '%s'"
- print(fmt % (full_tag, tag_prefix))
- return None, dirty
- tag = full_tag[len(tag_prefix):]
+ # commit: short hex revision ID
+ pieces["short"] = mo.group(3)
- # distance: number of commits since tag
- distance = int(mo.group(2))
+ else:
+ # HEX: no tags
+ pieces["closest-tag"] = None
+ count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
+ cwd=root)
+ pieces["distance"] = int(count_out) # total number of commits
+
+ # commit date: see ISO-8601 comment in git_versions_from_keywords()
+ date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
+ cwd=root)[0].strip()
+ pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
+
+ return pieces
+
+
+def plus_or_dot(pieces):
+ """Return a + if we don't already have one, else return a ."""
+ if "+" in pieces.get("closest-tag", ""):
+ return "."
+ return "+"
+
+
+def render_pep440(pieces):
+ """Build up version string, with post-release "local version identifier".
+
+ Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+ get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+ Exceptions:
+ 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"] or pieces["dirty"]:
+ rendered += plus_or_dot(pieces)
+ rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
+ if pieces["dirty"]:
+ rendered += ".dirty"
+ else:
+ # exception #1
+ rendered = "0+untagged.%d.g%s" % (pieces["distance"],
+ pieces["short"])
+ if pieces["dirty"]:
+ rendered += ".dirty"
+ return rendered
+
+
+def render_pep440_pre(pieces):
+ """TAG[.post.devDISTANCE] -- No -dirty.
+
+ Exceptions:
+ 1: no tags. 0.post.devDISTANCE
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"]:
+ rendered += ".post.dev%d" % pieces["distance"]
+ else:
+ # exception #1
+ rendered = "0.post.dev%d" % pieces["distance"]
+ return rendered
+
+
+def render_pep440_post(pieces):
+ """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+ The ".dev0" means dirty. Note that .dev0 sorts backwards
+ (a dirty tree will appear "older" than the corresponding clean one),
+ but you shouldn't be releasing software with -dirty anyways.
+
+ Exceptions:
+ 1: no tags. 0.postDISTANCE[.dev0]
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"] or pieces["dirty"]:
+ rendered += ".post%d" % pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ rendered += plus_or_dot(pieces)
+ rendered += "g%s" % pieces["short"]
+ else:
+ # exception #1
+ rendered = "0.post%d" % pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ rendered += "+g%s" % pieces["short"]
+ return rendered
+
+
+def render_pep440_old(pieces):
+ """TAG[.postDISTANCE[.dev0]] .
+
+ The ".dev0" means dirty.
+
+ Eexceptions:
+ 1: no tags. 0.postDISTANCE[.dev0]
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"] or pieces["dirty"]:
+ rendered += ".post%d" % pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ else:
+ # exception #1
+ rendered = "0.post%d" % pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ return rendered
- # commit: short hex revision ID
- commit = mo.group(3)
- # now build up version string, with post-release "local version
- # identifier". Our goal: TAG[+NUM.gHEX[.dirty]] . Note that if you get a
- # tagged build and then dirty it, you'll get TAG+0.gHEX.dirty . So you
- # can always test version.endswith(".dirty").
- version = tag
- if distance or dirty:
- version += "+%d.g%s" % (distance, commit) + dirty_suffix
+def render_git_describe(pieces):
+ """TAG[-DISTANCE-gHEX][-dirty].
- return version, dirty
+ Like 'git describe --tags --dirty --always'.
+ Exceptions:
+ 1: no tags. HEX[-dirty] (note: no 'g' prefix)
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"]:
+ rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+ else:
+ # exception #1
+ rendered = pieces["short"]
+ if pieces["dirty"]:
+ rendered += "-dirty"
+ return rendered
-def git_versions_from_vcs(tag_prefix, root, verbose=False):
- # this runs 'git' from the root of the source tree. This only gets called
- # if the git-archive 'subst' keywords were *not* expanded, and
- # _version.py hasn't already been rewritten with a short version string,
- # meaning we're inside a checked out source tree.
- if not os.path.exists(os.path.join(root, ".git")):
- if verbose:
- print("no .git in %s" % root)
- return {} # get_versions() will try next method
+def render_git_describe_long(pieces):
+ """TAG-DISTANCE-gHEX[-dirty].
- GITS = ["git"]
- if sys.platform == "win32":
- GITS = ["git.cmd", "git.exe"]
- # if there is a tag, this yields TAG-NUM-gHEX[-dirty]
- # if there are no tags, this yields HEX[-dirty] (no NUM)
- stdout = run_command(GITS, ["describe", "--tags", "--dirty",
- "--always", "--long"],
- cwd=root)
- # --long was added in git-1.5.5
- if stdout is None:
- return {} # try next method
- version, dirty = git_parse_vcs_describe(stdout, tag_prefix, verbose)
-
- # build "full", which is FULLHEX[.dirty]
- stdout = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
- if stdout is None:
- return {}
- full = stdout.strip()
- if dirty:
- full += ".dirty"
+ Like 'git describe --tags --dirty --always -long'.
+ The distance/hash is unconditional.
- return {"version": version, "full": full}
+ Exceptions:
+ 1: no tags. HEX[-dirty] (note: no 'g' prefix)
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+ else:
+ # exception #1
+ rendered = pieces["short"]
+ if pieces["dirty"]:
+ rendered += "-dirty"
+ return rendered
+
+
+def render(pieces, style):
+ """Render the given version pieces into the requested style."""
+ if pieces["error"]:
+ return {"version": "unknown",
+ "full-revisionid": pieces.get("long"),
+ "dirty": None,
+ "error": pieces["error"],
+ "date": None}
+
+ if not style or style == "default":
+ style = "pep440" # the default
+
+ if style == "pep440":
+ rendered = render_pep440(pieces)
+ elif style == "pep440-pre":
+ rendered = render_pep440_pre(pieces)
+ elif style == "pep440-post":
+ rendered = render_pep440_post(pieces)
+ elif style == "pep440-old":
+ rendered = render_pep440_old(pieces)
+ elif style == "git-describe":
+ rendered = render_git_describe(pieces)
+ elif style == "git-describe-long":
+ rendered = render_git_describe_long(pieces)
+ else:
+ raise ValueError("unknown style '%s'" % style)
+
+ return {"version": rendered, "full-revisionid": pieces["long"],
+ "dirty": pieces["dirty"], "error": None,
+ "date": pieces.get("date")}
-def get_versions(default={"version": "0+unknown", "full": ""}, verbose=False):
+def get_versions():
+ """Get version information or return default if unable to do so."""
# I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
# __file__, we can work backwards from there to the root. Some
# py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
# case we can only use expanded keywords.
- keywords = {"refnames": git_refnames, "full": git_full}
- ver = git_versions_from_keywords(keywords, tag_prefix, verbose)
- if ver:
- return ver
+ cfg = get_config()
+ verbose = cfg.verbose
+
+ try:
+ return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
+ verbose)
+ except NotThisMethod:
+ pass
try:
root = os.path.realpath(__file__)
# versionfile_source is the relative path from the top of the source
# tree (where the .git directory might live) to this file. Invert
# this to find the root from __file__.
- for i in versionfile_source.split('/'):
+ for i in cfg.versionfile_source.split('/'):
root = os.path.dirname(root)
except NameError:
- return default
+ return {"version": "0+unknown", "full-revisionid": None,
+ "dirty": None,
+ "error": "unable to find root of source tree",
+ "date": None}
+
+ try:
+ pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
+ return render(pieces, cfg.style)
+ except NotThisMethod:
+ pass
+
+ try:
+ if cfg.parentdir_prefix:
+ return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+ except NotThisMethod:
+ pass
- return (git_versions_from_vcs(tag_prefix, root, verbose)
- or versions_from_parentdir(parentdir_prefix, root, verbose)
- or default)
+ return {"version": "0+unknown", "full-revisionid": None,
+ "dirty": None,
+ "error": "unable to compute version", "date": None}
diff --git a/screed/conversion.py b/screed/conversion.py
index 3ae21db..26382dc 100644
--- a/screed/conversion.py
+++ b/screed/conversion.py
@@ -1,6 +1,6 @@
-from __future__ import absolute_import
-# Copyright (c) 2008-2010, Michigan State University
+# Copyright (c) 2008-2010, Michigan State University.
+from __future__ import absolute_import
from .openscreed import ScreedDB
_MAXLINELEN = 80
@@ -54,7 +54,7 @@ def ToFastq(dbFile, outputFile):
outFile = open(outputFile, 'wb')
db = ScreedDB(dbFile)
- for value in db.itervalues():
+ for n, value in enumerate(db.itervalues()):
line = '@%s %s\n%s\n+\n%s\n' % (value['name'],
GetComments(value),
linewrap(str(value['sequence'])),
@@ -63,6 +63,8 @@ def ToFastq(dbFile, outputFile):
db.close()
outFile.close()
+ return n + 1
+
def ToFasta(dbFile, outputFile):
"""
@@ -72,10 +74,12 @@ def ToFasta(dbFile, outputFile):
outFile = open(outputFile, 'wb')
db = ScreedDB(dbFile)
- for value in db.itervalues():
+ for n, value in enumerate(db.itervalues()):
line = '>%s %s\n%s\n' % (value['name'], GetComments(value),
linewrap(str(value['sequence'])))
outFile.write(line.encode('UTF-8'))
db.close()
outFile.close()
+
+ return n + 1
diff --git a/screed/createscreed.py b/screed/createscreed.py
index d09447d..c32d231 100644
--- a/screed/createscreed.py
+++ b/screed/createscreed.py
@@ -1,8 +1,18 @@
+# Copyright (c) 2016, The Regents of the University of California.
+
from __future__ import absolute_import
-from . import DBConstants
+
+import argparse
+import itertools
import os
-import sqlite3
+try:
+ import sqlite3
+except ImportError:
+ pass
import itertools
+import sys
+
+from . import DBConstants, fasta, fastq, openscreed
def create_db(filepath, fields, rcrditer):
@@ -12,6 +22,12 @@ def create_db(filepath, fields, rcrditer):
record. rcrditer is an iterator returning records over a
sequence dataset. Records yielded are in dictionary form
"""
+ try:
+ sqlite3
+ except NameError:
+ raise Exception("error: sqlite3 is needed for this functionality" +
+ " but is not installed.")
+
if not filepath.endswith(DBConstants.fileExtension):
filepath += DBConstants.fileExtension
@@ -81,3 +97,34 @@ def create_db(filepath, fields, rcrditer):
con.commit()
con.close()
+
+
+def make_db(filename):
+ iterfunc = openscreed.Open(filename, parse_description=True)
+
+ field_mapping = {
+ fastq.fastq_iter.__name__: fastq.FieldTypes,
+ fasta.fasta_iter.__name__: fasta.FieldTypes
+ }
+
+ fieldTypes = field_mapping[iterfunc.iter_fn.__name__]
+
+ # Create the screed db
+ create_db(filename, fieldTypes, iterfunc)
+
+
+def main(args):
+ parser = argparse.ArgumentParser(description="A shell interface to the "
+ "screed database writing function")
+ parser.add_argument('filename')
+ args = parser.parse_args(args)
+
+ make_db(args.filename)
+
+ print("Database saved in {}{}".format(args.filename,
+ DBConstants.fileExtension))
+ exit(0)
+
+
+if __name__ == "__main__":
+ main(sys.argv[1:])
diff --git a/screed/dna.py b/screed/dna.py
index 77d0dd8..25c2041 100644
--- a/screed/dna.py
+++ b/screed/dna.py
@@ -1,3 +1,5 @@
+# Copyright (c) 2016, The Regents of the University of California.
+
import array
import string
@@ -32,17 +34,14 @@ def reverse_complement(s):
rc = reverse_complement # alias 'rc' to 'reverse_complement'
-try:
- __complementTranslation = str.maketrans('ACTG', 'TGAC')
-except AttributeError:
- __complementTranslation = string.maketrans('ACTG', 'TGAC')
+__complementTranslation = {"A": "T", "C": "G", "G": "C", "T": "A", "N": "N"}
def complement(s):
"""
Return complement of 's'.
"""
- c = s.translate(__complementTranslation)
+ c = "".join(__complementTranslation[n] for n in s)
return c
diff --git a/screed/dump_fasta.py b/screed/dump_fasta.py
new file mode 100644
index 0000000..5483145
--- /dev/null
+++ b/screed/dump_fasta.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2008, Michigan State University.
+# Copyright (c) 2016, The Regents of the University of California.
+
+from __future__ import print_function
+
+import argparse
+import os
+import sys
+
+from screed import ToFasta
+
+
+# Shell interface to the ToFasta screed conversion function
+def main(args):
+ parser = argparse.ArgumentParser(
+ description="Convert a screed database to a FASTA file")
+ parser.add_argument('dbfile')
+ parser.add_argument('outputfile', default='/dev/stdout', nargs='?')
+ args = parser.parse_args(args)
+
+ if not os.path.isfile(args.dbfile):
+ print("No such file: %s" % args.dbfile)
+ exit(1)
+
+ n = ToFasta(args.dbfile, args.outputfile)
+
+ sys.stderr.write('Wrote {} records in FASTA format.\n'.format(n))
+
+
+if __name__ == '__main__':
+ main(sys.argv[1])
diff --git a/screed/dump_fastq.py b/screed/dump_fastq.py
new file mode 100644
index 0000000..d63e5d7
--- /dev/null
+++ b/screed/dump_fastq.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2008, Michigan State University.
+# Copyright (c) 2016, The Regents of the University of California.
+
+from __future__ import print_function
+from screed import ToFastq
+import argparse
+import sys
+import os
+
+
+# Shell interface to the ToFastq screed conversion function
+def main(args):
+ parser = argparse.ArgumentParser(
+ description="Convert a screed database to a FASTA file")
+ parser.add_argument('dbfile')
+ parser.add_argument('outputfile', default='/dev/stdout', nargs='?')
+ args = parser.parse_args(args)
+
+ if not os.path.isfile(args.dbfile):
+ print("No such file: %s" % args.dbfile)
+ exit(1)
+
+ n = ToFastq(args.dbfile, args.outputfile)
+
+ sys.stderr.write('Wrote {} records in FASTQ format.\n'.format(n))
+
+
+if __name__ == '__main__':
+ main(sys.argv[1])
diff --git a/screed/dump_to_fasta.py b/screed/dump_to_fasta.py
deleted file mode 100644
index aa58e51..0000000
--- a/screed/dump_to_fasta.py
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) 2008-2010, Michigan State University
-
-from __future__ import print_function
-from screed import ToFasta
-import sys
-import os
-
-# Shell interface to the ToFasta screed conversion function
-if __name__ == '__main__':
- if len(sys.argv) != 3:
- print("Usage: %s <dbfilename> <outputfilename>" % sys.argv[0])
- exit(1)
-
- dbFile = sys.argv[1]
- outputFile = sys.argv[2]
-
- if not os.path.isfile(dbFile):
- print("No such file: %s" % dbFile)
- exit(1)
- if os.path.isfile(outputFile):
- os.unlink(outputFile)
-
- ToFasta(dbFile, outputFile)
diff --git a/screed/dump_to_fastq.py b/screed/dump_to_fastq.py
deleted file mode 100644
index 8551295..0000000
--- a/screed/dump_to_fastq.py
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) 2008-2010, Michigan State University
-
-from __future__ import print_function
-from screed import ToFastq
-import sys
-import os
-
-# Shell interface to the ToFastq screed conversion function
-if __name__ == '__main__':
- if len(sys.argv) != 3:
- print("Usage: %s <dbfilename> <outputfilename>" % sys.argv[0])
- exit(1)
-
- dbFile = sys.argv[1]
- outputFile = sys.argv[2]
-
- if not os.path.isfile(dbFile):
- print("No such file: %s" % dbFile)
- exit(1)
- if os.path.isfile(outputFile):
- os.unlink(outputFile)
-
- ToFastq(dbFile, outputFile)
diff --git a/screed/fadbm.py b/screed/fadbm.py
deleted file mode 100755
index 0c8db4a..0000000
--- a/screed/fadbm.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) 2008-2010, Michigan State University
-
-import sys
-from __init__ import read_fasta_sequences
-import DBConstants
-
-# A shell interface to the screed FADBM database writing function
-if __name__ == "__main__":
- # Make sure the user entered the command line arguments correctly
- if len(sys.argv) != 2:
- sys.stderr.write("ERROR: USAGE IS: %s <dbfilename>\n" % sys.argv[0])
- exit(1)
-
- filename = sys.argv[1]
- read_fasta_sequences(filename)
-
- print("Database saved in %s%s" % (sys.argv[1], DBConstants.fileExtension))
diff --git a/screed/fasta.py b/screed/fasta.py
index 07200ef..6febde3 100644
--- a/screed/fasta.py
+++ b/screed/fasta.py
@@ -1,7 +1,8 @@
-from __future__ import absolute_import
+# Copyright (c) 2016, The Regents of the University of California.
+from __future__ import absolute_import
from . import DBConstants
-from .screedRecord import Record, _Writer
+from .screedRecord import Record
from .utils import to_str
FieldTypes = (('name', DBConstants._INDEXED_TEXT_KEY),
@@ -18,7 +19,7 @@ def fasta_iter(handle, parse_description=False, line=None):
line = handle.readline()
while line:
- data = Record()
+ data = {}
line = to_str(line.strip())
if not line.startswith('>'):
@@ -45,12 +46,4 @@ def fasta_iter(handle, parse_description=False, line=None):
line = to_str(handle.readline())
data['sequence'] = ''.join(sequenceList)
- yield data
-
-
-class FASTA_Writer(_Writer):
-
- def write(self, record):
- s = ">%s %s\n%s\n" % (record.name, record.description,
- record.sequence,)
- self.fp.write(s)
+ yield Record(**data)
diff --git a/screed/fastq.py b/screed/fastq.py
index b14bc6c..bc9093d 100644
--- a/screed/fastq.py
+++ b/screed/fastq.py
@@ -1,7 +1,8 @@
-from __future__ import absolute_import
+# Copyright (c) 2016, The Regents of the University of California.
+from __future__ import absolute_import
from . import DBConstants
-from .screedRecord import Record, _Writer
+from .screedRecord import Record
from .utils import to_str
FieldTypes = (('name', DBConstants._INDEXED_TEXT_KEY),
@@ -19,7 +20,7 @@ def fastq_iter(handle, line=None, parse_description=False):
line = handle.readline()
line = to_str(line.strip())
while line:
- data = Record()
+ data = {}
if line and not line.startswith('@'):
raise IOError("Bad FASTQ format: no '@' at beginning of line")
@@ -60,12 +61,4 @@ def fastq_iter(handle, line=None, parse_description=False):
raise IOError('sequence and quality strings must be '
'of equal length')
- yield data
-
-
-class FASTQ_Writer(_Writer):
-
- def write(self, record):
- s = "@%s %s\n%s\n+\n%s\n" % (record.name, record.description,
- record.sequence, record.quality)
- self.fp.write(s)
+ yield Record(**data)
diff --git a/screed/fqdbm.py b/screed/fqdbm.py
deleted file mode 100755
index 4c3f4e7..0000000
--- a/screed/fqdbm.py
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) 2008-2010, Michigan State University
-
-from __future__ import absolute_import
-
-import sys
-from screed import read_fastq_sequences
-from screed import DBConstants
-
-# A shell interface to the screed FQDBM database writing function
-if __name__ == "__main__":
- # Make sure the user entered the command line arguments correctly
- if len(sys.argv) != 2:
- sys.stderr.write("ERROR: USAGE IS: %s <dbfilename>\n" % sys.argv[0])
- exit(1)
-
- filename = sys.argv[1]
- read_fastq_sequences(filename)
-
- print("Database saved in %s%s" % (sys.argv[1], DBConstants.fileExtension))
- exit(0)
diff --git a/screed/hava.py b/screed/hava.py
index 1d5a6f8..a0f4d3f 100644
--- a/screed/hava.py
+++ b/screed/hava.py
@@ -1,5 +1,6 @@
-from __future__ import absolute_import
+# Copyright (c) 2016, The Regents of the University of California.
+from __future__ import absolute_import
from . import DBConstants
from .utils import to_str
diff --git a/screed/openscreed.py b/screed/openscreed.py
index 765ec44..8344044 100644
--- a/screed/openscreed.py
+++ b/screed/openscreed.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2008-2015, Michigan State University
+# Copyright (c) 2008, Michigan State University.
"""Reader and writer for screed."""
from __future__ import absolute_import
@@ -6,7 +6,6 @@ from __future__ import absolute_import
import os
import io
import sys
-import sqlite3
import gzip
import bz2file
try:
@@ -15,26 +14,18 @@ except ImportError:
import UserDict
MutableMapping = UserDict.DictMixin
+try:
+ import sqlite3
+except ImportError:
+ pass
+
from . import DBConstants
from . import screedRecord
-from .fastq import fastq_iter, FASTQ_Writer
-from .fasta import fasta_iter, FASTA_Writer
+from .fastq import fastq_iter
+from .fasta import fasta_iter
from .utils import to_str
-def get_writer_class(read_iter):
- if read_iter.__name__ == 'fasta_iter':
- return FASTA_Writer
- elif read_iter.__name__ == 'fastq_iter':
- return FASTQ_Writer
-
-
-def open_writer(inp_filename, outp_filename):
- read_iter = open_reader(inp_filename)
- klass = get_writer_class(read_iter)
- return klass(outp_filename)
-
-
def _normalize_filename(filename):
"""Map '-' to '/dev/stdin' to handle the usual shortcut."""
if filename == '-':
@@ -122,11 +113,6 @@ class Open(object):
self.sequencefile.close()
-_open = open
-open = Open
-open_reader = open
-
-
class ScreedDB(MutableMapping):
"""
@@ -135,6 +121,12 @@ class ScreedDB(MutableMapping):
"""
def __init__(self, filepath):
+ try:
+ sqlite3
+ except NameError:
+ raise Exception("error: sqlite3 is needed for this " +
+ "functionality, but is not installed.")
+
self._filepath = filepath
self._db = None
if not self._filepath.endswith(DBConstants.fileExtension):
@@ -165,7 +157,7 @@ class ScreedDB(MutableMapping):
% self._filepath)
nothing = res.fetchone()
- if type(nothing) is not type(None):
+ if nothing is not None:
self._db.close()
raise TypeError("Database %s has too many tables." % filename)
@@ -215,7 +207,7 @@ class ScreedDB(MutableMapping):
DBConstants._DICT_TABLE,
self._queryBy)
res = cursor.execute(query, (key,))
- if type(res.fetchone()) is type(None):
+ if res.fetchone() is None:
raise KeyError("Key %s not found" % key)
return screedRecord._buildRecord(self.fields, self._db,
key,
@@ -244,7 +236,7 @@ class ScreedDB(MutableMapping):
DBConstants._DICT_TABLE,
DBConstants._PRIMARY_KEY)
res = cursor.execute(query, (index,))
- if type(res.fetchone()) is type(None):
+ if res.fetchone() is None:
raise KeyError("Index %d not found" % index)
return screedRecord._buildRecord(self.fields, self._db,
index,
diff --git a/screed/pygr_api.py b/screed/pygr_api.py
index 383f332..8defc64 100644
--- a/screed/pygr_api.py
+++ b/screed/pygr_api.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2008-2010, Michigan State University
+# Copyright (c) 2008, Michigan State University.
"""
A simple wrapper implementing a pygr-compatible SequenceDB based on screed.
diff --git a/screed/screedRecord.py b/screed/screedRecord.py
index ce34a7f..92eeb47 100644
--- a/screed/screedRecord.py
+++ b/screed/screedRecord.py
@@ -1,8 +1,12 @@
+# Copyright (c) 2016, The Regents of the University of California.
+
from __future__ import absolute_import
+from functools import total_ordering
import types
from . import DBConstants
import gzip
import bz2
+from io import BytesIO
try:
from collections import MutableMapping
@@ -16,8 +20,18 @@ class Record(MutableMapping):
Simple dict-like record interface with bag behavior.
"""
- def __init__(self, *args, **kwargs):
- self.d = dict(*args, **kwargs)
+ def __init__(self, name=None, sequence=None, **kwargs):
+ d = dict()
+ if name is not None:
+ d['name'] = name
+ if sequence is not None:
+ d['sequence'] = sequence
+
+ d.update(kwargs)
+
+ if 'quality' in d and d['quality'] is None:
+ del d['quality']
+ self.d = d
def __setitem__(self, name, value):
self.d[name] = value
@@ -36,11 +50,11 @@ class Record(MutableMapping):
def __getitem__(self, idx):
if isinstance(idx, slice):
- trimmed = Record(self.d)
+ trimmed = dict(self.d)
trimmed['sequence'] = trimmed['sequence'][idx]
if 'quality' in trimmed:
trimmed['quality'] = trimmed['quality'][idx]
- return Record(trimmed)
+ return Record(**trimmed)
return self.d[idx]
def __delitem__(self, key):
@@ -53,6 +67,7 @@ class Record(MutableMapping):
return repr(self.d)
+ at total_ordering
class _screed_attr(object):
"""
@@ -108,46 +123,20 @@ class _screed_attr(object):
"""
return "<%s '%s'>" % (self.__class__.__name__, self._attrName)
- def __cmp__(self, given):
- """
- Handles comparisons other than == and !=
- """
- ownString = __str__()
- if isinstance(given, _screed_attr):
- given = str(given)
- elif not isinstance(given, str):
- raise TypeError("Cannot compare to given type: %s" % type(given))
-
- if ownString < given:
- return -1
- elif ownString > given:
- return 1
- else:
- return 0
-
def __eq__(self, given):
"""
Compares attribute to given object in string form
"""
if isinstance(given, bytes):
return given == self.__str__()
-
- try:
+ else:
return str(given) == self.__str__()
- except AttributeError:
- raise TypeError("Cannot compare to given type: %s" % type(given))
- def __ne__(self, given):
- """
- Compares attribute to given object in string form
- """
+ def __lt__(self, given):
if isinstance(given, bytes):
- return self.__repr__() != given
-
- try:
- return self.__repr__() != str(given)
- except AttributeError:
- raise TypeError("Cannot compare to given type: %s" % type(given))
+ return self.__str__() < given
+ else:
+ return self.__str__() < str(given)
def __str__(self):
"""
@@ -201,26 +190,39 @@ def _buildRecord(fieldTuple, dbObj, rowName, queryBy):
else:
hackedResult.append((key, value))
- return Record(hackedResult)
-
-
-class _Writer(object):
-
- def __init__(self, filename, fp=None):
- self.filename = filename
- if fp is None:
- if filename.endswith('.gz'):
- fp = gzip.open(filename, 'w')
- elif filename.endswith('.bz2'):
- fp = bz2.BZ2File(filename, 'w')
- else:
- fp = file(filename, 'wb')
-
- self.fp = fp
-
- def consume(self, read_iter):
- for read in read_iter:
- self.write(read)
-
- def close(self):
- self.fp.close()
+ return Record(**dict(hackedResult))
+
+
+def write_fastx(record, fileobj):
+ """Write sequence record to 'fileobj' in FASTA/FASTQ format."""
+ isbytesio = isinstance(fileobj, BytesIO)
+ iswb = hasattr(fileobj, 'mode') and fileobj.mode == 'wb'
+ outputvalid = isbytesio or iswb
+ if not outputvalid:
+ message = ('cannot call "write_fastx" on object, must be of a file '
+ 'handle with mode "wb" or an instance of "BytesIO"')
+ raise AttributeError(message)
+
+ defline = record.name
+ if hasattr(record, 'description'):
+ defline += ' ' + record.description
+
+ if hasattr(record, 'quality'):
+ recstr = '@{defline}\n{sequence}\n+\n{quality}\n'.format(
+ defline=defline,
+ sequence=record.sequence,
+ quality=record.quality)
+ else:
+ recstr = '>{defline}\n{sequence}\n'.format(
+ defline=defline,
+ sequence=record.sequence)
+
+ fileobj.write(recstr.encode('utf-8'))
+
+
+def write_fastx_pair(read1, read2, fileobj):
+ """Write a pair of sequence records to 'fileobj' in FASTA/FASTQ format."""
+ if hasattr(read1, 'quality'):
+ assert hasattr(read2, 'quality')
+ write_record(read1, fileobj)
+ write_record(read2, fileobj)
diff --git a/screed/seqparse.py b/screed/seqparse.py
index 77f8bd7..2f4dbc3 100644
--- a/screed/seqparse.py
+++ b/screed/seqparse.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2008-2010, Michigan State University
+# Copyright (c) 2008, Michigan State University.
"""
seqparse contains custom sequence parsers for extending screed's
@@ -25,7 +25,7 @@ def read_fastq_sequences(filename):
Function to parse text from the given FASTQ file into a screed database
"""
# Will raise an exception if the file doesn't exist
- iterfunc = openscreed.open(filename, parse_description=True)
+ iterfunc = openscreed.Open(filename, parse_description=True)
# Create the screed db
create_db(filename, fastq.FieldTypes, iterfunc)
@@ -38,7 +38,7 @@ def read_fasta_sequences(filename):
Function to parse text from the given FASTA file into a screed database
"""
# Will raise an exception if the file doesn't exist
- iterfunc = openscreed.open(filename, parse_description=True)
+ iterfunc = openscreed.Open(filename, parse_description=True)
# Create the screed db
create_db(filename, fasta.FieldTypes, iterfunc)
diff --git a/screed/tests/__main__.py b/screed/tests/__main__.py
index 2f8ee30..10ad790 100644
--- a/screed/tests/__main__.py
+++ b/screed/tests/__main__.py
@@ -1,8 +1,17 @@
import os
import sys
+
if __name__ == '__main__':
- if len(sys.argv) == 1:
- sys.argv.append(os.path.dirname(__file__))
- import nose
- nose.main()
+ from setuptools import setup
+ setup_params = {
+ "setup_requires": ['pytest_runner'],
+ "tests_require": ['pytest'],
+ }
+ rootdir = os.path.dirname(os.path.dirname(__file__))
+ # TODO: read opts from pytest.ini
+ opts = '-m "not known_failing" -v'
+ sys.argv[1:] = ['pytest', '--addopts=' + opts]
+ os.chdir(rootdir)
+ setup(**setup_params)
+ sys.exit()
diff --git a/screed/tests/havaGen.py b/screed/tests/havaGen.py
index d58d402..1837ffb 100755
--- a/screed/tests/havaGen.py
+++ b/screed/tests/havaGen.py
@@ -12,7 +12,7 @@ marshoon
Since this 'sequence' has absolutely no utility outside of screed, it's only
purpose is to make sure screed can work with arbitrary fields when running
-the nosetests.
+the tests.
This is a work of fiction. Names are the product of the author's imagination
and any resemblance to real life is entirely coincidental.
diff --git a/screed/tests/screed_tst_utils.py b/screed/tests/screed_tst_utils.py
index f008d93..83a6b59 100644
--- a/screed/tests/screed_tst_utils.py
+++ b/screed/tests/screed_tst_utils.py
@@ -13,7 +13,6 @@ import os
import shutil
from pkg_resources import Requirement, resource_filename, ResolutionError
from io import StringIO
-import nose
import sys
import traceback
diff --git a/screed/tests/test_attriberror.py b/screed/tests/test_attriberror.py
new file mode 100644
index 0000000..8905376
--- /dev/null
+++ b/screed/tests/test_attriberror.py
@@ -0,0 +1,63 @@
+import screed
+from screed.DBConstants import fileExtension
+import os
+from . import screed_tst_utils as utils
+import shutil
+
+
+class nostring:
+ def __str__(self):
+ return ""
+
+ def __repr__(self):
+ return ""
+
+
+class test_comparisons():
+
+ def setup(self):
+ self._testfile = utils.get_temp_filename('test.fa')
+ shutil.copy(utils.get_test_data('test.fa'), self._testfile)
+ screed.read_fasta_sequences(self._testfile)
+
+ self._db = screed.ScreedDB(self._testfile)
+ self._ns = nostring()
+
+ def test_eq(self):
+ for k in self._db:
+ record = self._db.get(k)
+ res = (record.sequence == self._ns)
+ assert res is False, res
+
+ def test_neq(self):
+ for k in self._db:
+ record = self._db.get(k)
+ res = (record.sequence != self._ns)
+ assert res is True, res
+
+ def test_comp_greateq(self):
+ for k in self._db:
+ record = self._db.get(k)
+ res = (record.sequence >= self._ns)
+ assert res is True, res
+
+ def test_comp_lesseq(self):
+ for k in self._db:
+ record = self._db.get(k)
+ res = (record.sequence <= self._ns)
+ assert res is False, res
+
+ def test_comp_less(self):
+ for k in self._db:
+ record = self._db.get(k)
+ res = (record.sequence < self._ns)
+ assert res is False, res
+
+ def test_comp_great(self):
+ for k in self._db:
+ record = self._db.get(k)
+ res = (record.sequence > self._ns)
+ assert res is True, res
+
+ def teardown(self):
+ os.unlink(self._testfile + fileExtension)
diff --git a/screed/tests/test_db.py b/screed/tests/test_db.py
new file mode 100644
index 0000000..3edc743
--- /dev/null
+++ b/screed/tests/test_db.py
@@ -0,0 +1,76 @@
+import os
+import shutil
+
+import screed
+from screed.DBConstants import fileExtension
+from . import screed_tst_utils as utils
+
+
+def test_make_db():
+ _testfa = utils.get_temp_filename('test.fa')
+ shutil.copy(utils.get_test_data('test.fa'), _testfa)
+ screed.make_db(_testfa)
+
+ db = screed.ScreedDB(_testfa)
+
+ os.unlink(_testfa + fileExtension)
+
+
+def test_no_sqlite_openscreed():
+ import screed.openscreed
+
+ saveme = screed.openscreed.sqlite3
+ del screed.openscreed.sqlite3
+
+ try:
+ try:
+ screed.openscreed.ScreedDB('xxx')
+ except Exception as e:
+ assert 'sqlite3 is needed' in str(e)
+ finally:
+ screed.openscreed.sqlite3 = saveme
+
+
+def test_no_sqlite_createscreed():
+ import screed.createscreed
+
+ saveme = screed.createscreed.sqlite3
+ del screed.createscreed.sqlite3
+
+ try:
+ try:
+ screed.createscreed.create_db(None, None, None)
+ except Exception as e:
+ assert 'sqlite3 is needed' in str(e)
+ finally:
+ screed.createscreed.sqlite3 = saveme
+
+
+def test_nodb():
+ """
+ Tests if screed throws an appropriate exception if it is
+ asked to open a non-existant screed database
+ """
+ try:
+ db = screed.ScreedDB('foo')
+ assert 1 == 0 # Previous line should throw an error
+ except ValueError:
+ pass
+
+
+def test_wrongdb():
+ """
+ Tests if screed throws an appropriate exception if it is
+ asked to open a file that isn't a screed database
+ """
+ try:
+ blah = 'blah_screed'
+ blah_file = open(blah, 'wb')
+ blah_file.close()
+
+ db = screed.ScreedDB(blah)
+ os.unlink(blah)
+ assert 1 == 0
+ except TypeError:
+ os.unlink(blah)
+ pass
diff --git a/screed/tests/test_fasta.py b/screed/tests/test_fasta.py
index 20d67fc..0b11823 100644
--- a/screed/tests/test_fasta.py
+++ b/screed/tests/test_fasta.py
@@ -1,12 +1,19 @@
from __future__ import absolute_import, unicode_literals
import screed
from screed.DBConstants import fileExtension
+from screed.screedRecord import write_fastx
import os
from io import StringIO
+from io import BytesIO
from . import screed_tst_utils as utils
import shutil
+class FakeRecord(object):
+ """Empty extensible object"""
+ pass
+
+
def test_new_record():
# test for a bug where the record dict was not reset after each
# sequence load, leading to all records being identical if you
@@ -17,6 +24,7 @@ def test_new_record():
records = list(iter(screed.fasta.fasta_iter(s)))
assert records[0]['name'] == '1'
assert records[1]['name'] == '2'
+ assert not hasattr(records[0], 'accuracy') # check for legacy attribute
class Test_fasta(object):
@@ -117,40 +125,36 @@ class Test_fasta_whitespace(object):
os.unlink(self._testfa + fileExtension)
-def test_writer():
- fp = StringIO()
- w = screed.fasta.FASTA_Writer("", fp)
-
- class FakeRecord(object):
- pass
-
+def test_output_sans_desc():
read = FakeRecord()
read.name = 'foo'
- read.description = 'bar'
read.sequence = 'ATCG'
- w.write(read)
-
- assert fp.getvalue() == '>foo bar\nATCG\n'
-
-
-def test_writer_2():
- fp = StringIO()
- w = screed.fasta.FASTA_Writer("", fp)
+ fileobj = BytesIO()
+ write_fastx(read, fileobj)
+ assert fileobj.getvalue().decode('utf-8') == '>foo\nATCG\n'
- class FakeRecord(object):
- pass
+def test_output_with_desc():
read = FakeRecord()
read.name = 'foo'
read.description = 'bar'
read.sequence = 'ATCG'
- read_iter = [read]
+ fileobj = BytesIO()
+ write_fastx(read, fileobj)
+ assert fileobj.getvalue().decode('utf-8') == '>foo bar\nATCG\n'
- w.consume(read_iter)
- assert fp.getvalue() == '>foo bar\nATCG\n'
+def test_output_two_reads():
+ fileobj = BytesIO()
+ for i in range(2):
+ read = FakeRecord()
+ read.name = 'seq{}'.format(i)
+ read.sequence = 'GATTACA' * (i + 1)
+ write_fastx(read, fileobj)
+ testoutput = '>seq0\nGATTACA\n>seq1\nGATTACAGATTACA\n'
+ assert fileobj.getvalue().decode('utf-8') == testoutput
def test_fasta_slicing():
diff --git a/screed/tests/test_fastq.py b/screed/tests/test_fastq.py
index b84d7f6..bb99d2b 100644
--- a/screed/tests/test_fastq.py
+++ b/screed/tests/test_fastq.py
@@ -1,10 +1,18 @@
from __future__ import absolute_import, unicode_literals
import screed
from screed.DBConstants import fileExtension
+from screed.screedRecord import write_fastx
import os
from io import StringIO
+from io import BytesIO
from . import screed_tst_utils as utils
import shutil
+import pytest
+
+
+class FakeRecord(object):
+ """Empty extensible object"""
+ pass
def test_new_record():
@@ -118,42 +126,53 @@ class Test_fastq(object):
assert entry == self.db[entry.name]
-def test_writer():
- fp = StringIO()
- w = screed.fastq.FASTQ_Writer("", fp)
+def test_output_sans_desc():
+ read = FakeRecord()
+ read.name = 'foo'
+ read.sequence = 'ATCG'
+ read.quality = '####'
+
+ fileobj = BytesIO()
+ write_fastx(read, fileobj)
+ assert fileobj.getvalue().decode('utf-8') == '@foo\nATCG\n+\n####\n'
- class FakeRecord(object):
- pass
+def test_output_with_desc():
read = FakeRecord()
read.name = 'foo'
read.description = 'bar'
read.sequence = 'ATCG'
read.quality = '####'
- w.write(read)
-
- assert fp.getvalue() == '@foo bar\nATCG\n+\n####\n'
+ fileobj = BytesIO()
+ write_fastx(read, fileobj)
+ assert fileobj.getvalue().decode('utf-8') == '@foo bar\nATCG\n+\n####\n'
-def test_writer_2():
- fp = StringIO()
- w = screed.fastq.FASTQ_Writer("", fp)
+def test_output_two_reads():
+ fileobj = BytesIO()
+ for i in range(2):
+ read = FakeRecord()
+ read.name = 'seq{}'.format(i)
+ read.sequence = 'GATTACA' * (i + 1)
+ read.quality = '#######' * (i + 1)
+ write_fastx(read, fileobj)
+ testoutput = ('@seq0\nGATTACA\n+\n#######\n'
+ '@seq1\nGATTACAGATTACA\n+\n##############\n')
+ assert fileobj.getvalue().decode('utf-8') == testoutput
- class FakeRecord(object):
- pass
+def test_output_bad_mode():
read = FakeRecord()
read.name = 'foo'
read.description = 'bar'
read.sequence = 'ATCG'
read.quality = '####'
- read_iter = [read]
-
- w.consume(read_iter)
-
- assert fp.getvalue() == '@foo bar\nATCG\n+\n####\n'
+ fileobj = StringIO()
+ with pytest.raises(AttributeError) as ae:
+ write_fastx(read, fileobj)
+ assert 'cannot call "write_fastx" on object' in str(ae)
def test_fastq_slicing():
diff --git a/screed/tests/test_nodb.py b/screed/tests/test_nodb.py
deleted file mode 100644
index f4dd523..0000000
--- a/screed/tests/test_nodb.py
+++ /dev/null
@@ -1,33 +0,0 @@
-import screed
-import os
-from screed.DBConstants import fileExtension
-
-
-def test_nodb():
- """
- Tests if screed throws an appropriate exception if it is
- asked to open a non-existant screed database
- """
- try:
- db = screed.ScreedDB('foo')
- assert 1 == 0 # Previous line should throw an error
- except ValueError:
- pass
-
-
-def test_wrongdb():
- """
- Tests if screed throws an appropriate exception if it is
- asked to open a file that isn't a screed database
- """
- try:
- blah = 'blah_screed'
- blah_file = open(blah, 'wb')
- blah_file.close()
-
- db = screed.ScreedDB(blah)
- os.unlink(blah)
- assert 1 == 0
- except TypeError:
- os.unlink(blah)
- pass
diff --git a/screed/tests/test_open.py b/screed/tests/test_open.py
index 548b370..3d880ce 100644
--- a/screed/tests/test_open.py
+++ b/screed/tests/test_open.py
@@ -1,4 +1,7 @@
# Copyright (c) 2008-2015, Michigan State University
+"""
+Test `screed.open`.
+"""
from __future__ import absolute_import
@@ -90,16 +93,6 @@ def test_gz_open_fastq():
assert n > 0
-def test_get_writer_class_fasta():
- import screed.fasta
-
- filename = utils.get_test_data('test.fa')
-
- read_iter = screed.open(filename)
- x = screed.openscreed.get_writer_class(read_iter)
- assert x is screed.fasta.FASTA_Writer, x
-
-
def test_unknown_fileformat():
try:
diff --git a/screed/tests/test_open_cm.py b/screed/tests/test_open_cm.py
index 0009b0d..928cce3 100644
--- a/screed/tests/test_open_cm.py
+++ b/screed/tests/test_open_cm.py
@@ -1,4 +1,7 @@
# Copyright (c) 2008-2015, Michigan State University
+"""
+Test the use of `screed.open` as a ContextManager.
+"""
from . import screed_tst_utils as utils
import screed
@@ -78,16 +81,6 @@ def test_gz_open_fastq():
assert n > 0
-def test_get_writer_class_fasta():
- import screed.fasta
-
- filename = utils.get_test_data('test.fa')
-
- with screed.open(filename) as f:
- x = screed.openscreed.get_writer_class(f)
- assert x is screed.fasta.FASTA_Writer, x
-
-
def test_unknown_fileformat():
try:
with screed.open(__file__):
diff --git a/screed/tests/test_pygr_api.py b/screed/tests/test_pygr_api.py
index 0f37dd0..85933fb 100644
--- a/screed/tests/test_pygr_api.py
+++ b/screed/tests/test_pygr_api.py
@@ -1,21 +1,14 @@
-"""
-Test the pygr API.
-"""
-
from __future__ import absolute_import, unicode_literals
-try:
- import pygr
-except ImportError:
- import nose
- raise nose.SkipTest("pygr is required for these tests")
-
-import screed
-from screed.DBConstants import fileExtension
-from screed.pygr_api import ScreedSequenceDB, ScreedSequenceDB_ByIndex
-from pickle import dump, load
-from io import StringIO
-import os
+import pytest
+pygr = pytest.importorskip("pygr")
+
+import screed # nopep8
+from screed.DBConstants import fileExtension # nopep8
+from screed.pygr_api import ScreedSequenceDB, ScreedSequenceDB_ByIndex # nopep8
+from pickle import dump, load # nopep8
+from io import StringIO # nopep8
+import os # nopep8
testfa = os.path.join(os.path.dirname(__file__), 'test.fa')
diff --git a/screed/tests/test_record.py b/screed/tests/test_record.py
new file mode 100644
index 0000000..509f3ba
--- /dev/null
+++ b/screed/tests/test_record.py
@@ -0,0 +1,34 @@
+from __future__ import absolute_import, unicode_literals, print_function
+from screed import Record
+import pytest
+
+
+def test_create_quality_none():
+ r = Record(name='foo', sequence='ATGACG', quality=None)
+ assert not hasattr(r, 'quality')
+
+
+def test_len():
+ r = Record(name='foo', sequence='ATGACG')
+ assert len(r) == 6
+
+
+# copied over from khmer tests/test_read_parsers.py
+def test_read_type_basic():
+ name = "895:1:1:1246:14654 1:N:0:NNNNN"
+ sequence = "ACGT"
+ r = Record(name, sequence)
+
+ assert r.name == name
+ assert r.sequence == sequence
+ assert not hasattr(r, 'quality'), x
+ assert not hasattr(r, 'annotations'), x
+
+
+# copied over from khmer tests/test_read_parsers.py
+def test_read_type_attributes():
+ r = Record(sequence='ACGT', quality='good', name='1234', annotations='ann')
+ assert r.sequence == 'ACGT'
+ assert r.quality == 'good'
+ assert r.name == '1234'
+ assert r.annotations == 'ann'
diff --git a/screed/tests/test_shell.py b/screed/tests/test_shell.py
index c622dec..055da79 100644
--- a/screed/tests/test_shell.py
+++ b/screed/tests/test_shell.py
@@ -9,10 +9,53 @@ from . import screed_tst_utils as utils
import shutil
-class Test_fa_shell(test_fasta.Test_fasta):
+class Test_fa_shell_command(test_fasta.Test_fasta):
+ """
+ Tests the functionality of the 'db' command in creating a
+ screed database correctly from the shell
+ """
+
+ def setup(self):
+ thisdir = os.path.dirname(__file__)
+
+ self._testfa = utils.get_temp_filename('test.fa')
+ shutil.copy(utils.get_test_data('test.fa'), self._testfa)
+
+ cmd = ['screed', 'db', self._testfa]
+ ret = subprocess.check_call(cmd, stdout=subprocess.PIPE)
+ assert ret == 0, ret
+ self.db = screed.ScreedDB(self._testfa)
+
+ def teardown(self):
+ os.unlink(self._testfa + fileExtension)
+
+
+class Test_fq_shell_command(test_fastq.Test_fastq):
+
+ """
+ Tests the functionality of the 'db' command in creating a
+ screed database correctly from the shell
+ """
+
+ def setup(self):
+ thisdir = os.path.dirname(__file__)
+
+ self._testfq = utils.get_temp_filename('test.fastq')
+ shutil.copy(utils.get_test_data('test.fastq'), self._testfq)
+
+ cmd = ['screed', 'db', self._testfq]
+ ret = subprocess.check_call(cmd, stdout=subprocess.PIPE)
+ assert ret == 0, ret
+ self.db = screed.ScreedDB(self._testfq)
+
+ def teardown(self):
+ os.unlink(self._testfq + fileExtension)
+
+
+class Test_fa_shell_module(test_fasta.Test_fasta):
"""
- Tests the functionality of the script 'fadbm' in creating a
+ Tests the functionality of the 'db' command in creating a
screed database correctly from the shell
"""
@@ -22,19 +65,19 @@ class Test_fa_shell(test_fasta.Test_fasta):
self._testfa = utils.get_temp_filename('test.fa')
shutil.copy(utils.get_test_data('test.fa'), self._testfa)
- fadbm = os.path.join(thisdir, '..', 'fadbm.py')
- subprocess.check_call(['python', fadbm, self._testfa],
- stdout=subprocess.PIPE)
+ cmd = ['python', '-m', 'screed', 'db', self._testfa]
+ ret = subprocess.check_call(cmd, stdout=subprocess.PIPE)
+ assert ret == 0, ret
self.db = screed.ScreedDB(self._testfa)
def teardown(self):
os.unlink(self._testfa + fileExtension)
-class Test_fq_shell(test_fastq.Test_fastq):
+class Test_fq_shell_module(test_fastq.Test_fastq):
"""
- Tests the functionality of the script 'fqdbm' in creating a
+ Tests the functionality of the 'db' command in creating a
screed database correctly from the shell
"""
@@ -44,10 +87,55 @@ class Test_fq_shell(test_fastq.Test_fastq):
self._testfq = utils.get_temp_filename('test.fastq')
shutil.copy(utils.get_test_data('test.fastq'), self._testfq)
- fqdbm = os.path.join(thisdir, '..', 'fqdbm.py')
- subprocess.check_call(['python', fqdbm, self._testfq],
- stdout=subprocess.PIPE)
+ cmd = ['python', '-m', 'screed', 'db', self._testfq]
+ ret = subprocess.check_call(cmd, stdout=subprocess.PIPE)
+ assert ret == 0, ret
self.db = screed.ScreedDB(self._testfq)
def teardown(self):
os.unlink(self._testfq + fileExtension)
+
+
+class Test_convert_shell(test_fasta.Test_fasta):
+
+ """
+ Tests the ability to convert a fasta db to a fastq file, parse it into
+ a fastq db, save to a fasta file, parse the fasta file into a fasta
+ db and then run the fasta suite, all from the command line.
+ """
+
+ def setup(self):
+
+ self._fqName = utils.get_temp_filename('fa_to_fq')
+ self._faName = utils.get_temp_filename('fq_to_fa')
+ self._testfa = utils.get_temp_filename('test.fa')
+ shutil.copy(utils.get_test_data('test.fa'), self._testfa)
+
+ cmd = ['screed', 'db', self._testfa]
+ ret = subprocess.check_call(cmd, stdout=subprocess.PIPE)
+ assert ret == 0, ret
+
+ cmd = ['screed', 'dump_fastq', self._testfa, self._fqName]
+ ret = subprocess.check_call(cmd, stdout=subprocess.PIPE)
+ assert ret == 0, ret
+
+ cmd = ['screed', 'db', self._fqName]
+ ret = subprocess.check_call(cmd, stdout=subprocess.PIPE)
+ assert ret == 0, ret
+
+ cmd = ['screed', 'dump_fasta', self._fqName, self._faName]
+ ret = subprocess.check_call(cmd, stdout=subprocess.PIPE)
+ assert ret == 0, ret
+
+ cmd = ['screed', 'db', self._faName]
+ ret = subprocess.check_call(cmd, stdout=subprocess.PIPE)
+ assert ret == 0, ret
+
+ self.db = screed.ScreedDB(self._faName)
+
+ def teardown(self):
+ os.unlink(self._fqName)
+ os.unlink(self._fqName + fileExtension)
+ os.unlink(self._faName)
+ os.unlink(self._faName + fileExtension)
+ os.unlink(self._testfa + fileExtension)
diff --git a/screed/tests/test_streaming.py b/screed/tests/test_streaming.py
index a3ae342..cabecac 100644
--- a/screed/tests/test_streaming.py
+++ b/screed/tests/test_streaming.py
@@ -10,7 +10,7 @@ import io
import threading
import subprocess
-from nose.plugins.attrib import attr
+import pytest
import screed
from . import screed_tst_utils as utils
@@ -67,7 +67,7 @@ def test_stream_fq():
streamer(utils.get_test_data('test.fastq'))
- at attr('known_failing')
+ at pytest.mark.known_failing
def test_stream_fa_gz():
streamer(utils.get_test_data('test.fa.gz'))
@@ -80,7 +80,7 @@ def test_stream_gz_fail():
print(str(err))
- at attr('known_failing')
+ at pytest.mark.known_failing
def test_stream_fq_gz():
streamer(utils.get_test_data('test.fastq.gz'))
diff --git a/screed/utils.py b/screed/utils.py
index 006145b..35b1e87 100644
--- a/screed/utils.py
+++ b/screed/utils.py
@@ -1,3 +1,6 @@
+# Copyright (c) 2016, The Regents of the University of California.
+
+
def to_str(line):
try:
line = line.decode('utf-8')
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..6f7283a
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,16 @@
+[aliases]
+test=pytest
+
+[versioneer]
+VCS = git
+style = git-describe
+versionfile_source = screed/_version.py
+versionfile_build = screed/_version.py
+tag_prefix = v
+parentdir_prefix = '.'
+
+[tool:pytest]
+# If you change anything in addopts,
+# don't forget to update screed/tests/__main__.py too!
+addopts = -m "not known_failing" -v
+testpaths = screed/tests
diff --git a/setup.py b/setup.py
index 6fc12f4..2eeed38 100755
--- a/setup.py
+++ b/setup.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
+# Copyright (c) 2016, The Regents of the University of California.
from __future__ import print_function
try:
from setuptools import setup
@@ -6,30 +7,30 @@ except ImportError:
print('(WARNING: importing distutils, not setuptools!)')
from distutils.core import setup
-import imp
-fp, pathname, description = imp.find_module('versioneer')
-versioneer = imp.load_module('versioneer', fp, pathname, description)
-del imp
-versioneer.VCS = 'git'
-versioneer.versionfile_source = 'screed/_version.py'
-versioneer.versionfile_build = 'screed/_version.py'
-versioneer.tag_prefix = 'v' # i.e. v1.2.0
-versioneer.parentdir_prefix = '.'
-CMDCLASS = versioneer.get_cmdclass()
+import versioneer
+
setup(name='screed',
version=versioneer.get_version(),
- description='A short read database',
- author='Alex Nolley, C. Titus Brown',
- author_email='ctb at msu.edu',
+ cmdclass=versioneer.get_cmdclass(),
+ description='Screed is a biological sequence parsing and '
+ 'storage/retrieval library for DNA and protein sequences.',
+ author='Luiz Irber, Peter Cock, Michael R. Crusoe, Jacob Fenton, '
+ 'Thomas Fenzl, Sarah Guermond, Tim Head, Kevin D. Murray, '
+ 'Alexander Nolley, Camille Scott, Daniel Standage, '
+ 'Benjamin R. Taylor, Michael Wright, en zyme, C. Titus Brown',
+ author_email='ctbrown at ucdavis.edu',
url='http://github.com/dib-lab/screed/',
+ zip_safe=False,
include_package_data=True,
packages=['screed', 'screed.tests'],
package_data={
'screed.tests': ['test.*', 'test-whitespace.fa', 'empty.fa']},
license='BSD',
- test_suite='nose.collector',
- extras_require={'tests': ['nose >= 1.0']},
- cmdclass=versioneer.get_cmdclass(),
- install_requires=['bz2file']
- )
+ setup_requires=['pytest-runner'],
+ tests_require=['pytest >= 3.0', 'pytest-cov'],
+ install_requires=['bz2file'],
+ entry_points={'console_scripts': [
+ 'screed = screed.__main__:main'
+ ]
+ })
diff --git a/tox.ini b/tox.ini
index 57ef524..3911556 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,7 +1,17 @@
[tox]
-envlist = py27, py33, py34
+envlist = py27, py33, py34, py35
[testenv]
-commands = nosetests --with-xcoverage --with-xunit --cover-package=screed --cover-erase --attr '!known_failing'
+passenv = CI TRAVIS TRAVIS_*
+whitelist_externals = make
+commands =
+ make install-dependencies
+ pytest --cov -m 'not known_failing'
+ codecov
+ make pep8
+ make doc
deps =
- nosexcover
+ pytest
+ pytest-cov
+ codecov
+ sphinx
diff --git a/versioneer.py b/versioneer.py
index c00770f..64fea1c 100644
--- a/versioneer.py
+++ b/versioneer.py
@@ -1,7 +1,8 @@
-# Version: 0.14
+# Version: 0.18
+
+"""The Versioneer - like a rocketeer, but for versions.
-"""
The Versioneer
==============
@@ -9,7 +10,7 @@ The Versioneer
* https://github.com/warner/python-versioneer
* Brian Warner
* License: Public Domain
-* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, and pypy
+* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, and pypy
* [![Latest Version]
(https://pypip.in/version/versioneer/badge.svg?style=flat)
](https://pypi.python.org/pypi/versioneer/)
@@ -27,8 +28,8 @@ system, and maybe making new tarballs.
## Quick Install
* `pip install versioneer` to somewhere to your $PATH
-* run `versioneer-installer` in your source tree: this installs `versioneer.py`
-* follow the instructions below (also in the `versioneer.py` docstring)
+* add a `[versioneer]` section to your setup.cfg (see below)
+* run `versioneer install` in your source tree, commit the results
## Version Identifiers
@@ -57,7 +58,7 @@ unreleased software (between tags), the version identifier should provide
enough information to help developers recreate the same tree, while also
giving them an idea of roughly how old the tree is (after version 1.2, before
version 1.3). Many VCS systems can report a description that captures this,
-for example 'git describe --tags --dirty --always' reports things like
+for example `git describe --tags --dirty --always` reports things like
"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the
0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has
uncommitted changes.
@@ -71,194 +72,186 @@ The version identifier is used for multiple purposes:
Versioneer works by adding a special `_version.py` file into your source
tree, where your `__init__.py` can import it. This `_version.py` knows how to
-dynamically ask the VCS tool for version information at import time. However,
-when you use "setup.py build" or "setup.py sdist", `_version.py` in the new
-copy is replaced by a small static file that contains just the generated
-version data.
+dynamically ask the VCS tool for version information at import time.
`_version.py` also contains `$Revision$` markers, and the installation
process marks `_version.py` to have this marker rewritten with a tag name
-during the "git archive" command. As a result, generated tarballs will
+during the `git archive` command. As a result, generated tarballs will
contain enough information to get the proper version.
+To allow `setup.py` to compute a version too, a `versioneer.py` is added to
+the top level of your source tree, next to `setup.py` and the `setup.cfg`
+that configures it. This overrides several distutils/setuptools commands to
+compute the version when invoked, and changes `setup.py build` and `setup.py
+sdist` to replace `_version.py` with a small static file that contains just
+the generated version data.
## Installation
-First, decide on values for the following configuration variables:
+See [INSTALL.md](./INSTALL.md) for detailed installation instructions.
-* `VCS`: the version control system you use. Currently accepts "git".
+## Version-String Flavors
-* `versionfile_source`:
+Code which uses Versioneer can learn about its version string at runtime by
+importing `_version` from your main `__init__.py` file and running the
+`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can
+import the top-level `versioneer.py` and run `get_versions()`.
- A project-relative pathname into which the generated version strings should
- be written. This is usually a `_version.py` next to your project's main
- `__init__.py` file, so it can be imported at runtime. If your project uses
- `src/myproject/__init__.py`, this should be `src/myproject/_version.py`.
- This file should be checked in to your VCS as usual: the copy created below
- by `setup.py versioneer` will include code that parses expanded VCS
- keywords in generated tarballs. The 'build' and 'sdist' commands will
- replace it with a copy that has just the calculated version string.
+Both functions return a dictionary with different flavors of version
+information:
- This must be set even if your project does not have any modules (and will
- therefore never import `_version.py`), since "setup.py sdist" -based trees
- still need somewhere to record the pre-calculated version strings. Anywhere
- in the source tree should do. If there is a `__init__.py` next to your
- `_version.py`, the `setup.py versioneer` command (described below) will
- append some `__version__`-setting assignments, if they aren't already
- present.
+* `['version']`: A condensed version string, rendered using the selected
+ style. This is the most commonly used value for the project's version
+ string. The default "pep440" style yields strings like `0.11`,
+ `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section
+ below for alternative styles.
-* `versionfile_build`:
+* `['full-revisionid']`: detailed revision identifier. For Git, this is the
+ full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac".
- Like `versionfile_source`, but relative to the build directory instead of
- the source directory. These will differ when your setup.py uses
- 'package_dir='. If you have `package_dir={'myproject': 'src/myproject'}`,
- then you will probably have `versionfile_build='myproject/_version.py'` and
- `versionfile_source='src/myproject/_version.py'`.
+* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the
+ commit date in ISO 8601 format. This will be None if the date is not
+ available.
- If this is set to None, then `setup.py build` will not attempt to rewrite
- any `_version.py` in the built tree. If your project does not have any
- libraries (e.g. if it only builds a script), then you should use
- `versionfile_build = None` and override `distutils.command.build_scripts`
- to explicitly insert a copy of `versioneer.get_version()` into your
- generated script.
+* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that
+ this is only accurate if run in a VCS checkout, otherwise it is likely to
+ be False or None
-* `tag_prefix`:
+* `['error']`: if the version string could not be computed, this will be set
+ to a string describing the problem, otherwise it will be None. It may be
+ useful to throw an exception in setup.py if this is set, to avoid e.g.
+ creating tarballs with a version string of "unknown".
- a string, like 'PROJECTNAME-', which appears at the start of all VCS tags.
- If your tags look like 'myproject-1.2.0', then you should use
- tag_prefix='myproject-'. If you use unprefixed tags like '1.2.0', this
- should be an empty string.
+Some variants are more useful than others. Including `full-revisionid` in a
+bug report should allow developers to reconstruct the exact code being tested
+(or indicate the presence of local changes that should be shared with the
+developers). `version` is suitable for display in an "about" box or a CLI
+`--version` output: it can be easily compared against release notes and lists
+of bugs fixed in various releases.
-* `parentdir_prefix`:
+The installer adds the following text to your `__init__.py` to place a basic
+version in `YOURPROJECT.__version__`:
- a string, frequently the same as tag_prefix, which appears at the start of
- all unpacked tarball filenames. If your tarball unpacks into
- 'myproject-1.2.0', this should be 'myproject-'.
+ from ._version import get_versions
+ __version__ = get_versions()['version']
+ del get_versions
-This tool provides one script, named `versioneer-installer`. That script does
-one thing: write a copy of `versioneer.py` into the current directory.
+## Styles
+
+The setup.cfg `style=` configuration controls how the VCS information is
+rendered into a version string.
-To versioneer-enable your project:
+The default style, "pep440", produces a PEP440-compliant string, equal to the
+un-prefixed tag name for actual releases, and containing an additional "local
+version" section with more detail for in-between builds. For Git, this is
+TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags
+--dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the
+tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and
+that this commit is two revisions ("+2") beyond the "0.11" tag. For released
+software (exactly equal to a known tag), the identifier will only contain the
+stripped tag, e.g. "0.11".
-* 1: Run `versioneer-installer` to copy `versioneer.py` into the top of your
- source tree.
+Other styles are available. See [details.md](details.md) in the Versioneer
+source tree for descriptions.
-* 2: add the following lines to the top of your `setup.py`, with the
- configuration values you decided earlier:
+## Debugging
- ````
- import versioneer
- versioneer.VCS = 'git'
- versioneer.versionfile_source = 'src/myproject/_version.py'
- versioneer.versionfile_build = 'myproject/_version.py'
- versioneer.tag_prefix = '' # tags are like 1.2.0
- versioneer.parentdir_prefix = 'myproject-' # dirname like 'myproject-1.2.0'
- ````
+Versioneer tries to avoid fatal errors: if something goes wrong, it will tend
+to return a version of "0+unknown". To investigate the problem, run `setup.py
+version`, which will run the version-lookup code in a verbose mode, and will
+display the full contents of `get_versions()` (including the `error` string,
+which may help identify what went wrong).
-* 3: add the following arguments to the setup() call in your setup.py:
+## Known Limitations
- version=versioneer.get_version(),
- cmdclass=versioneer.get_cmdclass(),
+Some situations are known to cause problems for Versioneer. This details the
+most significant ones. More can be found on Github
+[issues page](https://github.com/warner/python-versioneer/issues).
-* 4: now run `setup.py versioneer`, which will create `_version.py`, and will
- modify your `__init__.py` (if one exists next to `_version.py`) to define
- `__version__` (by calling a function from `_version.py`). It will also
- modify your `MANIFEST.in` to include both `versioneer.py` and the generated
- `_version.py` in sdist tarballs.
+### Subprojects
-* 5: commit these changes to your VCS. To make sure you won't forget,
- `setup.py versioneer` will mark everything it touched for addition.
+Versioneer has limited support for source trees in which `setup.py` is not in
+the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are
+two common reasons why `setup.py` might not be in the root:
-## Post-Installation Usage
+* Source trees which contain multiple subprojects, such as
+ [Buildbot](https://github.com/buildbot/buildbot), which contains both
+ "master" and "slave" subprojects, each with their own `setup.py`,
+ `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI
+ distributions (and upload multiple independently-installable tarballs).
+* Source trees whose main purpose is to contain a C library, but which also
+ provide bindings to Python (and perhaps other langauges) in subdirectories.
-Once established, all uses of your tree from a VCS checkout should get the
-current version string. All generated tarballs should include an embedded
-version string (so users who unpack them will not need a VCS tool installed).
+Versioneer will look for `.git` in parent directories, and most operations
+should get the right version string. However `pip` and `setuptools` have bugs
+and implementation details which frequently cause `pip install .` from a
+subproject directory to fail to find a correct version string (so it usually
+defaults to `0+unknown`).
-If you distribute your project through PyPI, then the release process should
-boil down to two steps:
+`pip install --editable .` should work correctly. `setup.py install` might
+work too.
-* 1: git tag 1.0
-* 2: python setup.py register sdist upload
+Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in
+some later version.
-If you distribute it through github (i.e. users use github to generate
-tarballs with `git archive`), the process is:
+[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking
+this issue. The discussion in
+[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the
+issue from the Versioneer side in more detail.
+[pip PR#3176](https://github.com/pypa/pip/pull/3176) and
+[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve
+pip to let Versioneer work correctly.
-* 1: git tag 1.0
-* 2: git push; git push --tags
+Versioneer-0.16 and earlier only looked for a `.git` directory next to the
+`setup.cfg`, so subprojects were completely unsupported with those releases.
-Currently, all version strings must be based upon a tag. Versioneer will
-report "unknown" until your tree has at least one tag in its history. This
-restriction will be fixed eventually (see issue #12).
+### Editable installs with setuptools <= 18.5
-## Version-String Flavors
+`setup.py develop` and `pip install --editable .` allow you to install a
+project into a virtualenv once, then continue editing the source code (and
+test) without re-installing after every change.
-Code which uses Versioneer can learn about its version string at runtime by
-importing `_version` from your main `__init__.py` file and running the
-`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can
-import the top-level `versioneer.py` and run `get_versions()`.
+"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a
+convenient way to specify executable scripts that should be installed along
+with the python package.
-Both functions return a dictionary with different keys for different flavors
-of the version string:
-
-* `['version']`: A condensed PEP440-compliant string, equal to the
- un-prefixed tag name for actual releases, and containing an additional
- "local version" section with more detail for in-between builds. For Git,
- this is TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe
- --tags --dirty --always`. For example "0.11+2.g1076c97.dirty" indicates
- that the tree is like the "1076c97" commit but has uncommitted changes
- (".dirty"), and that this commit is two revisions ("+2") beyond the "0.11"
- tag. For released software (exactly equal to a known tag), the identifier
- will only contain the stripped tag, e.g. "0.11".
-
-* `['full']`: detailed revision identifier. For Git, this is the full SHA1
- commit id, followed by ".dirty" if the tree contains uncommitted changes,
- e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac.dirty".
-
-Some variants are more useful than others. Including `full` in a bug report
-should allow developers to reconstruct the exact code being tested (or
-indicate the presence of local changes that should be shared with the
-developers). `version` is suitable for display in an "about" box or a CLI
-`--version` output: it can be easily compared against release notes and lists
-of bugs fixed in various releases.
+These both work as expected when using modern setuptools. When using
+setuptools-18.5 or earlier, however, certain operations will cause
+`pkg_resources.DistributionNotFound` errors when running the entrypoint
+script, which must be resolved by re-installing the package. This happens
+when the install happens with one version, then the egg_info data is
+regenerated while a different version is checked out. Many setup.py commands
+cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into
+a different virtualenv), so this can be surprising.
-The `setup.py versioneer` command adds the following text to your
-`__init__.py` to place a basic version in `YOURPROJECT.__version__`:
+[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes
+this one, but upgrading to a newer version of setuptools should probably
+resolve it.
+
+### Unicode version strings
+
+While Versioneer works (and is continually tested) with both Python 2 and
+Python 3, it is not entirely consistent with bytes-vs-unicode distinctions.
+Newer releases probably generate unicode version strings on py2. It's not
+clear that this is wrong, but it may be surprising for applications when then
+write these strings to a network connection or include them in bytes-oriented
+APIs like cryptographic checksums.
+
+[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates
+this question.
- from ._version import get_versions
- __version__ = get_versions()['version']
- del get_versions
## Updating Versioneer
To upgrade your project to a new release of Versioneer, do the following:
* install the new Versioneer (`pip install -U versioneer` or equivalent)
-* re-run `versioneer-installer` in your source tree to replace your copy of
- `versioneer.py`
-* edit `setup.py`, if necessary, to include any new configuration settings
- indicated by the release notes
-* re-run `setup.py versioneer` to replace `SRC/_version.py`
+* edit `setup.cfg`, if necessary, to include any new configuration settings
+ indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details.
+* re-run `versioneer install` in your source tree, to replace
+ `SRC/_version.py`
* commit any changed files
-### Upgrading from 0.10 to 0.11
-
-You must add a `versioneer.VCS = "git"` to your `setup.py` before re-running
-`setup.py versioneer`. This will enable the use of additional version-control
-systems (SVN, etc) in the future.
-
-### Upgrading from 0.11 to 0.12
-
-Nothing special.
-
-## Upgrading to 0.14
-
-0.14 changes the format of the version string. 0.13 and earlier used
-hyphen-separated strings like "0.11-2-g1076c97-dirty". 0.14 and beyond use a
-plus-separated "local version" section strings, with dot-separated
-components, like "0.11+2.g1076c97". PEP440-strict tools did not like the old
-format, but should be ok with the new one.
-
## Future Directions
This tool is designed to make it easily extended to other version-control
@@ -275,40 +268,130 @@ number of intermediate scripts.
## License
-To make Versioneer easier to embed, all its code is hereby released into the
-public domain. The `_version.py` that it creates is also in the public
-domain.
+To make Versioneer easier to embed, all its code is dedicated to the public
+domain. The `_version.py` that it creates is also in the public domain.
+Specifically, both are released under the Creative Commons "Public Domain
+Dedication" license (CC0-1.0), as described in
+https://creativecommons.org/publicdomain/zero/1.0/ .
"""
+from __future__ import print_function
+try:
+ import configparser
+except ImportError:
+ import ConfigParser as configparser
import errno
+import json
import os
import re
import subprocess
import sys
-from distutils.command.build import build as _build
-from distutils.command.sdist import sdist as _sdist
-from distutils.core import Command
-
-# these configuration settings will be overridden by setup.py after it
-# imports us
-versionfile_source = None
-versionfile_build = None
-tag_prefix = None
-parentdir_prefix = None
-VCS = None
+
+
+class VersioneerConfig:
+ """Container for Versioneer configuration parameters."""
+
+
+def get_root():
+ """Get the project root directory.
+
+ We require that all commands are run from the project root, i.e. the
+ directory that contains setup.py, setup.cfg, and versioneer.py .
+ """
+ root = os.path.realpath(os.path.abspath(os.getcwd()))
+ setup_py = os.path.join(root, "setup.py")
+ versioneer_py = os.path.join(root, "versioneer.py")
+ if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)):
+ # allow 'python path/to/setup.py COMMAND'
+ root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0])))
+ setup_py = os.path.join(root, "setup.py")
+ versioneer_py = os.path.join(root, "versioneer.py")
+ if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)):
+ err = ("Versioneer was unable to run the project root directory. "
+ "Versioneer requires setup.py to be executed from "
+ "its immediate directory (like 'python setup.py COMMAND'), "
+ "or in a way that lets it use sys.argv[0] to find the root "
+ "(like 'python path/to/setup.py COMMAND').")
+ raise VersioneerBadRootError(err)
+ try:
+ # Certain runtime workflows (setup.py install/develop in a setuptools
+ # tree) execute all dependencies in a single python process, so
+ # "versioneer" may be imported multiple times, and python's shared
+ # module-import table will cache the first one. So we can't use
+ # os.path.dirname(__file__), as that will find whichever
+ # versioneer.py was first imported, even in later projects.
+ me = os.path.realpath(os.path.abspath(__file__))
+ me_dir = os.path.normcase(os.path.splitext(me)[0])
+ vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0])
+ if me_dir != vsr_dir:
+ print("Warning: build in %s is using versioneer.py from %s"
+ % (os.path.dirname(me), versioneer_py))
+ except NameError:
+ pass
+ return root
+
+
+def get_config_from_root(root):
+ """Read the project setup.cfg file to determine Versioneer config."""
+ # This might raise EnvironmentError (if setup.cfg is missing), or
+ # configparser.NoSectionError (if it lacks a [versioneer] section), or
+ # configparser.NoOptionError (if it lacks "VCS="). See the docstring at
+ # the top of versioneer.py for instructions on writing your setup.cfg .
+ setup_cfg = os.path.join(root, "setup.cfg")
+ parser = configparser.SafeConfigParser()
+ with open(setup_cfg, "r") as f:
+ parser.readfp(f)
+ VCS = parser.get("versioneer", "VCS") # mandatory
+
+ def get(parser, name):
+ if parser.has_option("versioneer", name):
+ return parser.get("versioneer", name)
+ return None
+ cfg = VersioneerConfig()
+ cfg.VCS = VCS
+ cfg.style = get(parser, "style") or ""
+ cfg.versionfile_source = get(parser, "versionfile_source")
+ cfg.versionfile_build = get(parser, "versionfile_build")
+ cfg.tag_prefix = get(parser, "tag_prefix")
+ if cfg.tag_prefix in ("''", '""'):
+ cfg.tag_prefix = ""
+ cfg.parentdir_prefix = get(parser, "parentdir_prefix")
+ cfg.verbose = get(parser, "verbose")
+ return cfg
+
+
+class NotThisMethod(Exception):
+ """Exception raised if a method is not valid for the current scenario."""
+
# these dictionaries contain VCS-specific tools
LONG_VERSION_PY = {}
+HANDLERS = {}
-def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
+def register_vcs_handler(vcs, method): # decorator
+ """Decorator to mark a method as the handler for a particular VCS."""
+ def decorate(f):
+ """Store f in HANDLERS[vcs][method]."""
+ if vcs not in HANDLERS:
+ HANDLERS[vcs] = {}
+ HANDLERS[vcs][method] = f
+ return f
+ return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
+ env=None):
+ """Call the given command(s)."""
assert isinstance(commands, list)
p = None
for c in commands:
try:
+ dispcmd = str([c] + args)
# remember shell=False, so use git.cmd on windows, not just git
- p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
+ p = subprocess.Popen([c] + args, cwd=cwd, env=env,
+ stdout=subprocess.PIPE,
stderr=(subprocess.PIPE if hide_stderr
else None))
break
@@ -317,21 +400,24 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
if e.errno == errno.ENOENT:
continue
if verbose:
- print("unable to run %s" % args[0])
+ print("unable to run %s" % dispcmd)
print(e)
- return None
+ return None, None
else:
if verbose:
print("unable to find command, tried %s" % (commands,))
- return None
+ return None, None
stdout = p.communicate()[0].strip()
if sys.version_info[0] >= 3:
stdout = stdout.decode()
if p.returncode != 0:
if verbose:
- print("unable to run %s (error)" % args[0])
- return None
- return stdout
+ print("unable to run %s (error)" % dispcmd)
+ print("stdout was %s" % stdout)
+ return None, p.returncode
+ return stdout, p.returncode
+
+
LONG_VERSION_PY['git'] = '''
# This file helps to compute a version number in source trees obtained from
# git-archive tarball (such as those provided by githubs download-from-tag
@@ -340,7 +426,9 @@ LONG_VERSION_PY['git'] = '''
# that just contains the computed version number.
# This file is released into the public domain. Generated by
-# versioneer-0.14 (https://github.com/warner/python-versioneer)
+# versioneer-0.18 (https://github.com/warner/python-versioneer)
+
+"""Git implementation of _version.py."""
import errno
import os
@@ -348,23 +436,68 @@ import re
import subprocess
import sys
-# these strings will be replaced by git during git-archive
-git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s"
-git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s"
-# these strings are filled in when 'setup.py versioneer' creates _version.py
-tag_prefix = "%(TAG_PREFIX)s"
-parentdir_prefix = "%(PARENTDIR_PREFIX)s"
-versionfile_source = "%(VERSIONFILE_SOURCE)s"
+def get_keywords():
+ """Get the keywords needed to look up the version information."""
+ # these strings will be replaced by git during git-archive.
+ # setup.py/versioneer.py will grep for the variable names, so they must
+ # each be defined on a line of their own. _version.py will just call
+ # get_keywords().
+ git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s"
+ git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s"
+ git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s"
+ keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
+ return keywords
+
+
+class VersioneerConfig:
+ """Container for Versioneer configuration parameters."""
+
+
+def get_config():
+ """Create, populate and return the VersioneerConfig() object."""
+ # these strings are filled in when 'setup.py versioneer' creates
+ # _version.py
+ cfg = VersioneerConfig()
+ cfg.VCS = "git"
+ cfg.style = "%(STYLE)s"
+ cfg.tag_prefix = "%(TAG_PREFIX)s"
+ cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s"
+ cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s"
+ cfg.verbose = False
+ return cfg
+
+
+class NotThisMethod(Exception):
+ """Exception raised if a method is not valid for the current scenario."""
-def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method): # decorator
+ """Decorator to mark a method as the handler for a particular VCS."""
+ def decorate(f):
+ """Store f in HANDLERS[vcs][method]."""
+ if vcs not in HANDLERS:
+ HANDLERS[vcs] = {}
+ HANDLERS[vcs][method] = f
+ return f
+ return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
+ env=None):
+ """Call the given command(s)."""
assert isinstance(commands, list)
p = None
for c in commands:
try:
+ dispcmd = str([c] + args)
# remember shell=False, so use git.cmd on windows, not just git
- p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
+ p = subprocess.Popen([c] + args, cwd=cwd, env=env,
+ stdout=subprocess.PIPE,
stderr=(subprocess.PIPE if hide_stderr
else None))
break
@@ -373,36 +506,52 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
if e.errno == errno.ENOENT:
continue
if verbose:
- print("unable to run %%s" %% args[0])
+ print("unable to run %%s" %% dispcmd)
print(e)
- return None
+ return None, None
else:
if verbose:
print("unable to find command, tried %%s" %% (commands,))
- return None
+ return None, None
stdout = p.communicate()[0].strip()
if sys.version_info[0] >= 3:
stdout = stdout.decode()
if p.returncode != 0:
if verbose:
- print("unable to run %%s (error)" %% args[0])
- return None
- return stdout
-
+ print("unable to run %%s (error)" %% dispcmd)
+ print("stdout was %%s" %% stdout)
+ return None, p.returncode
+ return stdout, p.returncode
+
+
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+ """Try to determine the version from the parent directory name.
+
+ Source tarballs conventionally unpack into a directory that includes both
+ the project name and a version string. We will also support searching up
+ two directory levels for an appropriately named parent directory
+ """
+ rootdirs = []
+
+ for i in range(3):
+ dirname = os.path.basename(root)
+ if dirname.startswith(parentdir_prefix):
+ return {"version": dirname[len(parentdir_prefix):],
+ "full-revisionid": None,
+ "dirty": False, "error": None, "date": None}
+ else:
+ rootdirs.append(root)
+ root = os.path.dirname(root) # up a level
-def versions_from_parentdir(parentdir_prefix, root, verbose=False):
- # Source tarballs conventionally unpack into a directory that includes
- # both the project name and a version string.
- dirname = os.path.basename(root)
- if not dirname.startswith(parentdir_prefix):
- if verbose:
- print("guessing rootdir is '%%s', but '%%s' doesn't start with "
- "prefix '%%s'" %% (root, dirname, parentdir_prefix))
- return None
- return {"version": dirname[len(parentdir_prefix):], "full": ""}
+ if verbose:
+ print("Tried directories %%s but none started with prefix %%s" %%
+ (str(rootdirs), parentdir_prefix))
+ raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+ at register_vcs_handler("git", "get_keywords")
def git_get_keywords(versionfile_abs):
+ """Extract version information from the given file."""
# the code embedded in _version.py can just fetch the value of these
# keywords. When used from setup.py, we don't want to import _version.py,
# so we do it with a regexp instead. This function is not used from
@@ -419,20 +568,35 @@ def git_get_keywords(versionfile_abs):
mo = re.search(r'=\s*"(.*)"', line)
if mo:
keywords["full"] = mo.group(1)
+ if line.strip().startswith("git_date ="):
+ mo = re.search(r'=\s*"(.*)"', line)
+ if mo:
+ keywords["date"] = mo.group(1)
f.close()
except EnvironmentError:
pass
return keywords
-def git_versions_from_keywords(keywords, tag_prefix, verbose=False):
+ at register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+ """Get version information from git keywords."""
if not keywords:
- return {} # keyword-finding function failed to find keywords
+ raise NotThisMethod("no keywords at all, weird")
+ date = keywords.get("date")
+ if date is not None:
+ # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant
+ # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601
+ # -like" string, which we must then edit to make compliant), because
+ # it's been around since git-1.5.3, and it's too difficult to
+ # discover which version we're using, or to work around using an
+ # older one.
+ date = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
refnames = keywords["refnames"].strip()
if refnames.startswith("$Format"):
if verbose:
print("keywords are unexpanded, not using")
- return {} # unexpanded, so not in an unpacked git-archive tarball
+ raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
refs = set([r.strip() for r in refnames.strip("()").split(",")])
# starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
# just "foo-1.0". If we see a "tag: " prefix, prefer those.
@@ -448,7 +612,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose=False):
# "stabilization", as well as "HEAD" and "master".
tags = set([r for r in refs if re.search(r'\d', r)])
if verbose:
- print("discarding '%%s', no digits" %% ",".join(refs-tags))
+ print("discarding '%%s', no digits" %% ",".join(refs - tags))
if verbose:
print("likely tags: %%s" %% ",".join(sorted(tags)))
for ref in sorted(tags):
@@ -458,123 +622,328 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose=False):
if verbose:
print("picking %%s" %% r)
return {"version": r,
- "full": keywords["full"].strip()}
+ "full-revisionid": keywords["full"].strip(),
+ "dirty": False, "error": None,
+ "date": date}
# no suitable tags, so version is "0+unknown", but full hex is still there
if verbose:
print("no suitable tags, using unknown + full revision id")
return {"version": "0+unknown",
- "full": keywords["full"].strip()}
+ "full-revisionid": keywords["full"].strip(),
+ "dirty": False, "error": "no suitable tags", "date": None}
-def git_parse_vcs_describe(git_describe, tag_prefix, verbose=False):
- # TAG-NUM-gHEX[-dirty] or HEX[-dirty] . TAG might have hyphens.
+ at register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+ """Get version from 'git describe' in the root of the source tree.
- # dirty
+ This only gets called if the git-archive 'subst' keywords were *not*
+ expanded, and _version.py hasn't already been rewritten with a short
+ version string, meaning we're inside a checked out source tree.
+ """
+ GITS = ["git"]
+ if sys.platform == "win32":
+ GITS = ["git.cmd", "git.exe"]
+
+ out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
+ hide_stderr=True)
+ if rc != 0:
+ if verbose:
+ print("Directory %%s not under git control" %% root)
+ raise NotThisMethod("'git rev-parse --git-dir' returned error")
+
+ # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+ # if there isn't one, this yields HEX[-dirty] (no NUM)
+ describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
+ "--always", "--long",
+ "--match", "%%s*" %% tag_prefix],
+ cwd=root)
+ # --long was added in git-1.5.5
+ if describe_out is None:
+ raise NotThisMethod("'git describe' failed")
+ describe_out = describe_out.strip()
+ full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+ if full_out is None:
+ raise NotThisMethod("'git rev-parse' failed")
+ full_out = full_out.strip()
+
+ pieces = {}
+ pieces["long"] = full_out
+ pieces["short"] = full_out[:7] # maybe improved later
+ pieces["error"] = None
+
+ # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+ # TAG might have hyphens.
+ git_describe = describe_out
+
+ # look for -dirty suffix
dirty = git_describe.endswith("-dirty")
+ pieces["dirty"] = dirty
if dirty:
git_describe = git_describe[:git_describe.rindex("-dirty")]
- dirty_suffix = ".dirty" if dirty else ""
# now we have TAG-NUM-gHEX or HEX
- if "-" not in git_describe: # just HEX
- return "0+untagged.g"+git_describe+dirty_suffix, dirty
+ if "-" in git_describe:
+ # TAG-NUM-gHEX
+ mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+ if not mo:
+ # unparseable. Maybe git-describe is misbehaving?
+ pieces["error"] = ("unable to parse git-describe output: '%%s'"
+ %% describe_out)
+ return pieces
+
+ # tag
+ full_tag = mo.group(1)
+ if not full_tag.startswith(tag_prefix):
+ if verbose:
+ fmt = "tag '%%s' doesn't start with prefix '%%s'"
+ print(fmt %% (full_tag, tag_prefix))
+ pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'"
+ %% (full_tag, tag_prefix))
+ return pieces
+ pieces["closest-tag"] = full_tag[len(tag_prefix):]
- # just TAG-NUM-gHEX
- mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
- if not mo:
- # unparseable. Maybe git-describe is misbehaving?
- return "0+unparseable"+dirty_suffix, dirty
+ # distance: number of commits since tag
+ pieces["distance"] = int(mo.group(2))
- # tag
- full_tag = mo.group(1)
- if not full_tag.startswith(tag_prefix):
- if verbose:
- fmt = "tag '%%s' doesn't start with prefix '%%s'"
- print(fmt %% (full_tag, tag_prefix))
- return None, dirty
- tag = full_tag[len(tag_prefix):]
+ # commit: short hex revision ID
+ pieces["short"] = mo.group(3)
+
+ else:
+ # HEX: no tags
+ pieces["closest-tag"] = None
+ count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
+ cwd=root)
+ pieces["distance"] = int(count_out) # total number of commits
+
+ # commit date: see ISO-8601 comment in git_versions_from_keywords()
+ date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"],
+ cwd=root)[0].strip()
+ pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
+
+ return pieces
+
+
+def plus_or_dot(pieces):
+ """Return a + if we don't already have one, else return a ."""
+ if "+" in pieces.get("closest-tag", ""):
+ return "."
+ return "+"
+
+
+def render_pep440(pieces):
+ """Build up version string, with post-release "local version identifier".
+
+ Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+ get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+ Exceptions:
+ 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"] or pieces["dirty"]:
+ rendered += plus_or_dot(pieces)
+ rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"])
+ if pieces["dirty"]:
+ rendered += ".dirty"
+ else:
+ # exception #1
+ rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"],
+ pieces["short"])
+ if pieces["dirty"]:
+ rendered += ".dirty"
+ return rendered
+
+
+def render_pep440_pre(pieces):
+ """TAG[.post.devDISTANCE] -- No -dirty.
+
+ Exceptions:
+ 1: no tags. 0.post.devDISTANCE
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"]:
+ rendered += ".post.dev%%d" %% pieces["distance"]
+ else:
+ # exception #1
+ rendered = "0.post.dev%%d" %% pieces["distance"]
+ return rendered
+
+
+def render_pep440_post(pieces):
+ """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+ The ".dev0" means dirty. Note that .dev0 sorts backwards
+ (a dirty tree will appear "older" than the corresponding clean one),
+ but you shouldn't be releasing software with -dirty anyways.
+
+ Exceptions:
+ 1: no tags. 0.postDISTANCE[.dev0]
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"] or pieces["dirty"]:
+ rendered += ".post%%d" %% pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ rendered += plus_or_dot(pieces)
+ rendered += "g%%s" %% pieces["short"]
+ else:
+ # exception #1
+ rendered = "0.post%%d" %% pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ rendered += "+g%%s" %% pieces["short"]
+ return rendered
+
+
+def render_pep440_old(pieces):
+ """TAG[.postDISTANCE[.dev0]] .
+
+ The ".dev0" means dirty.
+
+ Eexceptions:
+ 1: no tags. 0.postDISTANCE[.dev0]
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"] or pieces["dirty"]:
+ rendered += ".post%%d" %% pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ else:
+ # exception #1
+ rendered = "0.post%%d" %% pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ return rendered
- # distance: number of commits since tag
- distance = int(mo.group(2))
- # commit: short hex revision ID
- commit = mo.group(3)
+def render_git_describe(pieces):
+ """TAG[-DISTANCE-gHEX][-dirty].
- # now build up version string, with post-release "local version
- # identifier". Our goal: TAG[+NUM.gHEX[.dirty]] . Note that if you get a
- # tagged build and then dirty it, you'll get TAG+0.gHEX.dirty . So you
- # can always test version.endswith(".dirty").
- version = tag
- if distance or dirty:
- version += "+%%d.g%%s" %% (distance, commit) + dirty_suffix
+ Like 'git describe --tags --dirty --always'.
- return version, dirty
+ Exceptions:
+ 1: no tags. HEX[-dirty] (note: no 'g' prefix)
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"]:
+ rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"])
+ else:
+ # exception #1
+ rendered = pieces["short"]
+ if pieces["dirty"]:
+ rendered += "-dirty"
+ return rendered
-def git_versions_from_vcs(tag_prefix, root, verbose=False):
- # this runs 'git' from the root of the source tree. This only gets called
- # if the git-archive 'subst' keywords were *not* expanded, and
- # _version.py hasn't already been rewritten with a short version string,
- # meaning we're inside a checked out source tree.
+def render_git_describe_long(pieces):
+ """TAG-DISTANCE-gHEX[-dirty].
- if not os.path.exists(os.path.join(root, ".git")):
- if verbose:
- print("no .git in %%s" %% root)
- return {} # get_versions() will try next method
+ Like 'git describe --tags --dirty --always -long'.
+ The distance/hash is unconditional.
- GITS = ["git"]
- if sys.platform == "win32":
- GITS = ["git.cmd", "git.exe"]
- # if there is a tag, this yields TAG-NUM-gHEX[-dirty]
- # if there are no tags, this yields HEX[-dirty] (no NUM)
- stdout = run_command(GITS, ["describe", "--tags", "--dirty",
- "--always", "--long"],
- cwd=root)
- # --long was added in git-1.5.5
- if stdout is None:
- return {} # try next method
- version, dirty = git_parse_vcs_describe(stdout, tag_prefix, verbose)
-
- # build "full", which is FULLHEX[.dirty]
- stdout = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
- if stdout is None:
- return {}
- full = stdout.strip()
- if dirty:
- full += ".dirty"
+ Exceptions:
+ 1: no tags. HEX[-dirty] (note: no 'g' prefix)
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"])
+ else:
+ # exception #1
+ rendered = pieces["short"]
+ if pieces["dirty"]:
+ rendered += "-dirty"
+ return rendered
+
+
+def render(pieces, style):
+ """Render the given version pieces into the requested style."""
+ if pieces["error"]:
+ return {"version": "unknown",
+ "full-revisionid": pieces.get("long"),
+ "dirty": None,
+ "error": pieces["error"],
+ "date": None}
+
+ if not style or style == "default":
+ style = "pep440" # the default
+
+ if style == "pep440":
+ rendered = render_pep440(pieces)
+ elif style == "pep440-pre":
+ rendered = render_pep440_pre(pieces)
+ elif style == "pep440-post":
+ rendered = render_pep440_post(pieces)
+ elif style == "pep440-old":
+ rendered = render_pep440_old(pieces)
+ elif style == "git-describe":
+ rendered = render_git_describe(pieces)
+ elif style == "git-describe-long":
+ rendered = render_git_describe_long(pieces)
+ else:
+ raise ValueError("unknown style '%%s'" %% style)
- return {"version": version, "full": full}
+ return {"version": rendered, "full-revisionid": pieces["long"],
+ "dirty": pieces["dirty"], "error": None,
+ "date": pieces.get("date")}
-def get_versions(default={"version": "0+unknown", "full": ""}, verbose=False):
+def get_versions():
+ """Get version information or return default if unable to do so."""
# I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
# __file__, we can work backwards from there to the root. Some
# py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
# case we can only use expanded keywords.
- keywords = {"refnames": git_refnames, "full": git_full}
- ver = git_versions_from_keywords(keywords, tag_prefix, verbose)
- if ver:
- return ver
+ cfg = get_config()
+ verbose = cfg.verbose
+
+ try:
+ return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
+ verbose)
+ except NotThisMethod:
+ pass
try:
root = os.path.realpath(__file__)
# versionfile_source is the relative path from the top of the source
# tree (where the .git directory might live) to this file. Invert
# this to find the root from __file__.
- for i in versionfile_source.split('/'):
+ for i in cfg.versionfile_source.split('/'):
root = os.path.dirname(root)
except NameError:
- return default
+ return {"version": "0+unknown", "full-revisionid": None,
+ "dirty": None,
+ "error": "unable to find root of source tree",
+ "date": None}
- return (git_versions_from_vcs(tag_prefix, root, verbose)
- or versions_from_parentdir(parentdir_prefix, root, verbose)
- or default)
+ try:
+ pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
+ return render(pieces, cfg.style)
+ except NotThisMethod:
+ pass
+
+ try:
+ if cfg.parentdir_prefix:
+ return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+ except NotThisMethod:
+ pass
+
+ return {"version": "0+unknown", "full-revisionid": None,
+ "dirty": None,
+ "error": "unable to compute version", "date": None}
'''
+ at register_vcs_handler("git", "get_keywords")
def git_get_keywords(versionfile_abs):
+ """Extract version information from the given file."""
# the code embedded in _version.py can just fetch the value of these
# keywords. When used from setup.py, we don't want to import _version.py,
# so we do it with a regexp instead. This function is not used from
@@ -591,20 +960,35 @@ def git_get_keywords(versionfile_abs):
mo = re.search(r'=\s*"(.*)"', line)
if mo:
keywords["full"] = mo.group(1)
+ if line.strip().startswith("git_date ="):
+ mo = re.search(r'=\s*"(.*)"', line)
+ if mo:
+ keywords["date"] = mo.group(1)
f.close()
except EnvironmentError:
pass
return keywords
-def git_versions_from_keywords(keywords, tag_prefix, verbose=False):
+ at register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+ """Get version information from git keywords."""
if not keywords:
- return {} # keyword-finding function failed to find keywords
+ raise NotThisMethod("no keywords at all, weird")
+ date = keywords.get("date")
+ if date is not None:
+ # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
+ # datestamp. However we prefer "%ci" (which expands to an "ISO-8601
+ # -like" string, which we must then edit to make compliant), because
+ # it's been around since git-1.5.3, and it's too difficult to
+ # discover which version we're using, or to work around using an
+ # older one.
+ date = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
refnames = keywords["refnames"].strip()
if refnames.startswith("$Format"):
if verbose:
print("keywords are unexpanded, not using")
- return {} # unexpanded, so not in an unpacked git-archive tarball
+ raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
refs = set([r.strip() for r in refnames.strip("()").split(",")])
# starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
# just "foo-1.0". If we see a "tag: " prefix, prefer those.
@@ -620,7 +1004,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose=False):
# "stabilization", as well as "HEAD" and "master".
tags = set([r for r in refs if re.search(r'\d', r)])
if verbose:
- print("discarding '%s', no digits" % ",".join(refs-tags))
+ print("discarding '%s', no digits" % ",".join(refs - tags))
if verbose:
print("likely tags: %s" % ",".join(sorted(tags)))
for ref in sorted(tags):
@@ -630,96 +1014,115 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose=False):
if verbose:
print("picking %s" % r)
return {"version": r,
- "full": keywords["full"].strip()}
+ "full-revisionid": keywords["full"].strip(),
+ "dirty": False, "error": None,
+ "date": date}
# no suitable tags, so version is "0+unknown", but full hex is still there
if verbose:
print("no suitable tags, using unknown + full revision id")
return {"version": "0+unknown",
- "full": keywords["full"].strip()}
+ "full-revisionid": keywords["full"].strip(),
+ "dirty": False, "error": "no suitable tags", "date": None}
+
+ at register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+ """Get version from 'git describe' in the root of the source tree.
-def git_parse_vcs_describe(git_describe, tag_prefix, verbose=False):
- # TAG-NUM-gHEX[-dirty] or HEX[-dirty] . TAG might have hyphens.
+ This only gets called if the git-archive 'subst' keywords were *not*
+ expanded, and _version.py hasn't already been rewritten with a short
+ version string, meaning we're inside a checked out source tree.
+ """
+ GITS = ["git"]
+ if sys.platform == "win32":
+ GITS = ["git.cmd", "git.exe"]
- # dirty
+ out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
+ hide_stderr=True)
+ if rc != 0:
+ if verbose:
+ print("Directory %s not under git control" % root)
+ raise NotThisMethod("'git rev-parse --git-dir' returned error")
+
+ # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+ # if there isn't one, this yields HEX[-dirty] (no NUM)
+ describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
+ "--always", "--long",
+ "--match", "%s*" % tag_prefix],
+ cwd=root)
+ # --long was added in git-1.5.5
+ if describe_out is None:
+ raise NotThisMethod("'git describe' failed")
+ describe_out = describe_out.strip()
+ full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+ if full_out is None:
+ raise NotThisMethod("'git rev-parse' failed")
+ full_out = full_out.strip()
+
+ pieces = {}
+ pieces["long"] = full_out
+ pieces["short"] = full_out[:7] # maybe improved later
+ pieces["error"] = None
+
+ # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+ # TAG might have hyphens.
+ git_describe = describe_out
+
+ # look for -dirty suffix
dirty = git_describe.endswith("-dirty")
+ pieces["dirty"] = dirty
if dirty:
git_describe = git_describe[:git_describe.rindex("-dirty")]
- dirty_suffix = ".dirty" if dirty else ""
# now we have TAG-NUM-gHEX or HEX
- if "-" not in git_describe: # just HEX
- return "0+untagged.g"+git_describe+dirty_suffix, dirty
-
- # just TAG-NUM-gHEX
- mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
- if not mo:
- # unparseable. Maybe git-describe is misbehaving?
- return "0+unparseable"+dirty_suffix, dirty
-
- # tag
- full_tag = mo.group(1)
- if not full_tag.startswith(tag_prefix):
- if verbose:
- fmt = "tag '%s' doesn't start with prefix '%s'"
- print(fmt % (full_tag, tag_prefix))
- return None, dirty
- tag = full_tag[len(tag_prefix):]
-
- # distance: number of commits since tag
- distance = int(mo.group(2))
-
- # commit: short hex revision ID
- commit = mo.group(3)
-
- # now build up version string, with post-release "local version
- # identifier". Our goal: TAG[+NUM.gHEX[.dirty]] . Note that if you get a
- # tagged build and then dirty it, you'll get TAG+0.gHEX.dirty . So you
- # can always test version.endswith(".dirty").
- version = tag
- if distance or dirty:
- version += "+%d.g%s" % (distance, commit) + dirty_suffix
-
- return version, dirty
+ if "-" in git_describe:
+ # TAG-NUM-gHEX
+ mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+ if not mo:
+ # unparseable. Maybe git-describe is misbehaving?
+ pieces["error"] = ("unable to parse git-describe output: '%s'"
+ % describe_out)
+ return pieces
+
+ # tag
+ full_tag = mo.group(1)
+ if not full_tag.startswith(tag_prefix):
+ if verbose:
+ fmt = "tag '%s' doesn't start with prefix '%s'"
+ print(fmt % (full_tag, tag_prefix))
+ pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
+ % (full_tag, tag_prefix))
+ return pieces
+ pieces["closest-tag"] = full_tag[len(tag_prefix):]
+ # distance: number of commits since tag
+ pieces["distance"] = int(mo.group(2))
-def git_versions_from_vcs(tag_prefix, root, verbose=False):
- # this runs 'git' from the root of the source tree. This only gets called
- # if the git-archive 'subst' keywords were *not* expanded, and
- # _version.py hasn't already been rewritten with a short version string,
- # meaning we're inside a checked out source tree.
+ # commit: short hex revision ID
+ pieces["short"] = mo.group(3)
- if not os.path.exists(os.path.join(root, ".git")):
- if verbose:
- print("no .git in %s" % root)
- return {} # get_versions() will try next method
+ else:
+ # HEX: no tags
+ pieces["closest-tag"] = None
+ count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
+ cwd=root)
+ pieces["distance"] = int(count_out) # total number of commits
- GITS = ["git"]
- if sys.platform == "win32":
- GITS = ["git.cmd", "git.exe"]
- # if there is a tag, this yields TAG-NUM-gHEX[-dirty]
- # if there are no tags, this yields HEX[-dirty] (no NUM)
- stdout = run_command(GITS, ["describe", "--tags", "--dirty",
- "--always", "--long"],
- cwd=root)
- # --long was added in git-1.5.5
- if stdout is None:
- return {} # try next method
- version, dirty = git_parse_vcs_describe(stdout, tag_prefix, verbose)
-
- # build "full", which is FULLHEX[.dirty]
- stdout = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
- if stdout is None:
- return {}
- full = stdout.strip()
- if dirty:
- full += ".dirty"
+ # commit date: see ISO-8601 comment in git_versions_from_keywords()
+ date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
+ cwd=root)[0].strip()
+ pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
- return {"version": version, "full": full}
+ return pieces
def do_vcs_install(manifest_in, versionfile_source, ipy):
+ """Git-specific installation logic for Versioneer.
+
+ For Git, this means creating/changing .gitattributes to mark _version.py
+ for export-subst keyword substitution.
+ """
GITS = ["git"]
if sys.platform == "win32":
GITS = ["git.cmd", "git.exe"]
@@ -752,201 +1155,537 @@ def do_vcs_install(manifest_in, versionfile_source, ipy):
run_command(GITS, ["add", "--"] + files)
-def versions_from_parentdir(parentdir_prefix, root, verbose=False):
- # Source tarballs conventionally unpack into a directory that includes
- # both the project name and a version string.
- dirname = os.path.basename(root)
- if not dirname.startswith(parentdir_prefix):
- if verbose:
- print("guessing rootdir is '%s', but '%s' doesn't start with "
- "prefix '%s'" % (root, dirname, parentdir_prefix))
- return None
- return {"version": dirname[len(parentdir_prefix):], "full": ""}
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+ """Try to determine the version from the parent directory name.
+
+ Source tarballs conventionally unpack into a directory that includes both
+ the project name and a version string. We will also support searching up
+ two directory levels for an appropriately named parent directory
+ """
+ rootdirs = []
+
+ for i in range(3):
+ dirname = os.path.basename(root)
+ if dirname.startswith(parentdir_prefix):
+ return {"version": dirname[len(parentdir_prefix):],
+ "full-revisionid": None,
+ "dirty": False, "error": None, "date": None}
+ else:
+ rootdirs.append(root)
+ root = os.path.dirname(root) # up a level
+
+ if verbose:
+ print("Tried directories %s but none started with prefix %s" %
+ (str(rootdirs), parentdir_prefix))
+ raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+
SHORT_VERSION_PY = """
-# This file was generated by 'versioneer.py' (0.14) from
+# This file was generated by 'versioneer.py' (0.18) from
# revision-control system data, or from the parent directory name of an
# unpacked source archive. Distribution tarballs contain a pre-generated copy
# of this file.
-version_version = '%(version)s'
-version_full = '%(full)s'
-def get_versions(default={}, verbose=False):
- return {'version': version_version, 'full': version_full}
+import json
-"""
+version_json = '''
+%s
+''' # END VERSION_JSON
-DEFAULT = {"version": "0+unknown", "full": "unknown"}
+
+def get_versions():
+ return json.loads(version_json)
+"""
def versions_from_file(filename):
- versions = {}
+ """Try to determine the version from _version.py if present."""
try:
with open(filename) as f:
- for line in f.readlines():
- mo = re.match("version_version = '([^']+)'", line)
- if mo:
- versions["version"] = mo.group(1)
- mo = re.match("version_full = '([^']+)'", line)
- if mo:
- versions["full"] = mo.group(1)
+ contents = f.read()
except EnvironmentError:
- return {}
-
- return versions
+ raise NotThisMethod("unable to read _version.py")
+ mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON",
+ contents, re.M | re.S)
+ if not mo:
+ mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON",
+ contents, re.M | re.S)
+ if not mo:
+ raise NotThisMethod("no version_json in _version.py")
+ return json.loads(mo.group(1))
def write_to_version_file(filename, versions):
+ """Write the given version number to the given _version.py file."""
+ os.unlink(filename)
+ contents = json.dumps(versions, sort_keys=True,
+ indent=1, separators=(",", ": "))
with open(filename, "w") as f:
- f.write(SHORT_VERSION_PY % versions)
+ f.write(SHORT_VERSION_PY % contents)
print("set %s to '%s'" % (filename, versions["version"]))
-def get_root():
- try:
- return os.path.dirname(os.path.abspath(__file__))
- except NameError:
- return os.path.dirname(os.path.abspath(sys.argv[0]))
+def plus_or_dot(pieces):
+ """Return a + if we don't already have one, else return a ."""
+ if "+" in pieces.get("closest-tag", ""):
+ return "."
+ return "+"
-def vcs_function(vcs, suffix):
- return getattr(sys.modules[__name__], '%s_%s' % (vcs, suffix), None)
+def render_pep440(pieces):
+ """Build up version string, with post-release "local version identifier".
+ Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+ get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+ Exceptions:
+ 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"] or pieces["dirty"]:
+ rendered += plus_or_dot(pieces)
+ rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
+ if pieces["dirty"]:
+ rendered += ".dirty"
+ else:
+ # exception #1
+ rendered = "0+untagged.%d.g%s" % (pieces["distance"],
+ pieces["short"])
+ if pieces["dirty"]:
+ rendered += ".dirty"
+ return rendered
+
+
+def render_pep440_pre(pieces):
+ """TAG[.post.devDISTANCE] -- No -dirty.
+
+ Exceptions:
+ 1: no tags. 0.post.devDISTANCE
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"]:
+ rendered += ".post.dev%d" % pieces["distance"]
+ else:
+ # exception #1
+ rendered = "0.post.dev%d" % pieces["distance"]
+ return rendered
+
+
+def render_pep440_post(pieces):
+ """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+ The ".dev0" means dirty. Note that .dev0 sorts backwards
+ (a dirty tree will appear "older" than the corresponding clean one),
+ but you shouldn't be releasing software with -dirty anyways.
+
+ Exceptions:
+ 1: no tags. 0.postDISTANCE[.dev0]
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"] or pieces["dirty"]:
+ rendered += ".post%d" % pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ rendered += plus_or_dot(pieces)
+ rendered += "g%s" % pieces["short"]
+ else:
+ # exception #1
+ rendered = "0.post%d" % pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ rendered += "+g%s" % pieces["short"]
+ return rendered
+
+
+def render_pep440_old(pieces):
+ """TAG[.postDISTANCE[.dev0]] .
+
+ The ".dev0" means dirty.
+
+ Eexceptions:
+ 1: no tags. 0.postDISTANCE[.dev0]
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"] or pieces["dirty"]:
+ rendered += ".post%d" % pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ else:
+ # exception #1
+ rendered = "0.post%d" % pieces["distance"]
+ if pieces["dirty"]:
+ rendered += ".dev0"
+ return rendered
+
+
+def render_git_describe(pieces):
+ """TAG[-DISTANCE-gHEX][-dirty].
+
+ Like 'git describe --tags --dirty --always'.
+
+ Exceptions:
+ 1: no tags. HEX[-dirty] (note: no 'g' prefix)
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ if pieces["distance"]:
+ rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+ else:
+ # exception #1
+ rendered = pieces["short"]
+ if pieces["dirty"]:
+ rendered += "-dirty"
+ return rendered
+
+
+def render_git_describe_long(pieces):
+ """TAG-DISTANCE-gHEX[-dirty].
+
+ Like 'git describe --tags --dirty --always -long'.
+ The distance/hash is unconditional.
+
+ Exceptions:
+ 1: no tags. HEX[-dirty] (note: no 'g' prefix)
+ """
+ if pieces["closest-tag"]:
+ rendered = pieces["closest-tag"]
+ rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+ else:
+ # exception #1
+ rendered = pieces["short"]
+ if pieces["dirty"]:
+ rendered += "-dirty"
+ return rendered
+
+
+def render(pieces, style):
+ """Render the given version pieces into the requested style."""
+ if pieces["error"]:
+ return {"version": "unknown",
+ "full-revisionid": pieces.get("long"),
+ "dirty": None,
+ "error": pieces["error"],
+ "date": None}
+
+ if not style or style == "default":
+ style = "pep440" # the default
+
+ if style == "pep440":
+ rendered = render_pep440(pieces)
+ elif style == "pep440-pre":
+ rendered = render_pep440_pre(pieces)
+ elif style == "pep440-post":
+ rendered = render_pep440_post(pieces)
+ elif style == "pep440-old":
+ rendered = render_pep440_old(pieces)
+ elif style == "git-describe":
+ rendered = render_git_describe(pieces)
+ elif style == "git-describe-long":
+ rendered = render_git_describe_long(pieces)
+ else:
+ raise ValueError("unknown style '%s'" % style)
+
+ return {"version": rendered, "full-revisionid": pieces["long"],
+ "dirty": pieces["dirty"], "error": None,
+ "date": pieces.get("date")}
+
+
+class VersioneerBadRootError(Exception):
+ """The project root directory is unknown or missing key files."""
+
+
+def get_versions(verbose=False):
+ """Get the project version from whatever source is available.
+
+ Returns dict with two keys: 'version' and 'full'.
+ """
+ if "versioneer" in sys.modules:
+ # see the discussion in cmdclass.py:get_cmdclass()
+ del sys.modules["versioneer"]
-def get_versions(default=DEFAULT, verbose=False):
- # returns dict with two keys: 'version' and 'full'
- assert versionfile_source is not None, \
- "please set versioneer.versionfile_source"
- assert tag_prefix is not None, "please set versioneer.tag_prefix"
- assert parentdir_prefix is not None, \
- "please set versioneer.parentdir_prefix"
- assert VCS is not None, "please set versioneer.VCS"
-
- # I am in versioneer.py, which must live at the top of the source tree,
- # which we use to compute the root directory. py2exe/bbfreeze/non-CPython
- # don't have __file__, in which case we fall back to sys.argv[0] (which
- # ought to be the setup.py script). We prefer __file__ since that's more
- # robust in cases where setup.py was invoked in some weird way (e.g. pip)
root = get_root()
- versionfile_abs = os.path.join(root, versionfile_source)
+ cfg = get_config_from_root(root)
+
+ assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg"
+ handlers = HANDLERS.get(cfg.VCS)
+ assert handlers, "unrecognized VCS '%s'" % cfg.VCS
+ verbose = verbose or cfg.verbose
+ assert cfg.versionfile_source is not None, \
+ "please set versioneer.versionfile_source"
+ assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix"
+
+ versionfile_abs = os.path.join(root, cfg.versionfile_source)
- # extract version from first of _version.py, VCS command (e.g. 'git
+ # extract version from first of: _version.py, VCS command (e.g. 'git
# describe'), parentdir. This is meant to work for developers using a
# source checkout, for users of a tarball created by 'setup.py sdist',
# and for users of a tarball/zipball created by 'git archive' or github's
# download-from-tag feature or the equivalent in other VCSes.
- get_keywords_f = vcs_function(VCS, "get_keywords")
- versions_from_keywords_f = vcs_function(VCS, "versions_from_keywords")
- if get_keywords_f and versions_from_keywords_f:
- vcs_keywords = get_keywords_f(versionfile_abs)
- ver = versions_from_keywords_f(vcs_keywords, tag_prefix)
- if ver:
+ get_keywords_f = handlers.get("get_keywords")
+ from_keywords_f = handlers.get("keywords")
+ if get_keywords_f and from_keywords_f:
+ try:
+ keywords = get_keywords_f(versionfile_abs)
+ ver = from_keywords_f(keywords, cfg.tag_prefix, verbose)
if verbose:
print("got version from expanded keyword %s" % ver)
return ver
+ except NotThisMethod:
+ pass
- ver = versions_from_file(versionfile_abs)
- if ver:
+ try:
+ ver = versions_from_file(versionfile_abs)
if verbose:
print("got version from file %s %s" % (versionfile_abs, ver))
return ver
+ except NotThisMethod:
+ pass
- versions_from_vcs_f = vcs_function(VCS, "versions_from_vcs")
- if versions_from_vcs_f:
- ver = versions_from_vcs_f(tag_prefix, root, verbose)
- if ver:
+ from_vcs_f = handlers.get("pieces_from_vcs")
+ if from_vcs_f:
+ try:
+ pieces = from_vcs_f(cfg.tag_prefix, root, verbose)
+ ver = render(pieces, cfg.style)
if verbose:
print("got version from VCS %s" % ver)
return ver
+ except NotThisMethod:
+ pass
- ver = versions_from_parentdir(parentdir_prefix, root, verbose)
- if ver:
- if verbose:
- print("got version from parentdir %s" % ver)
- return ver
+ try:
+ if cfg.parentdir_prefix:
+ ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+ if verbose:
+ print("got version from parentdir %s" % ver)
+ return ver
+ except NotThisMethod:
+ pass
if verbose:
- print("got version from default %s" % default)
- return default
+ print("unable to compute version")
+ return {"version": "0+unknown", "full-revisionid": None,
+ "dirty": None, "error": "unable to compute version",
+ "date": None}
-def get_version(verbose=False):
- return get_versions(verbose=verbose)["version"]
+def get_version():
+ """Get the short version string for this project."""
+ return get_versions()["version"]
-class cmd_version(Command):
- description = "report generated version string"
- user_options = []
- boolean_options = []
- def initialize_options(self):
- pass
+def get_cmdclass():
+ """Get the custom setuptools/distutils subclasses used by Versioneer."""
+ if "versioneer" in sys.modules:
+ del sys.modules["versioneer"]
+ # this fixes the "python setup.py develop" case (also 'install' and
+ # 'easy_install .'), in which subdependencies of the main project are
+ # built (using setup.py bdist_egg) in the same python process. Assume
+ # a main project A and a dependency B, which use different versions
+ # of Versioneer. A's setup.py imports A's Versioneer, leaving it in
+ # sys.modules by the time B's setup.py is executed, causing B to run
+ # with the wrong versioneer. Setuptools wraps the sub-dep builds in a
+ # sandbox that restores sys.modules to it's pre-build state, so the
+ # parent is protected against the child's "import versioneer". By
+ # removing ourselves from sys.modules here, before the child build
+ # happens, we protect the child from the parent's versioneer too.
+ # Also see https://github.com/warner/python-versioneer/issues/52
+
+ cmds = {}
+
+ # we add "version" to both distutils and setuptools
+ from distutils.core import Command
+
+ class cmd_version(Command):
+ description = "report generated version string"
+ user_options = []
+ boolean_options = []
+
+ def initialize_options(self):
+ pass
- def finalize_options(self):
- pass
+ def finalize_options(self):
+ pass
- def run(self):
- ver = get_version(verbose=True)
- print("Version is currently: %s" % ver)
+ def run(self):
+ vers = get_versions(verbose=True)
+ print("Version: %s" % vers["version"])
+ print(" full-revisionid: %s" % vers.get("full-revisionid"))
+ print(" dirty: %s" % vers.get("dirty"))
+ print(" date: %s" % vers.get("date"))
+ if vers["error"]:
+ print(" error: %s" % vers["error"])
+ cmds["version"] = cmd_version
+
+ # we override "build_py" in both distutils and setuptools
+ #
+ # most invocation pathways end up running build_py:
+ # distutils/build -> build_py
+ # distutils/install -> distutils/build ->..
+ # setuptools/bdist_wheel -> distutils/install ->..
+ # setuptools/bdist_egg -> distutils/install_lib -> build_py
+ # setuptools/install -> bdist_egg ->..
+ # setuptools/develop -> ?
+ # pip install:
+ # copies source tree to a tempdir before running egg_info/etc
+ # if .git isn't copied too, 'git describe' will fail
+ # then does setup.py bdist_wheel, or sometimes setup.py install
+ # setup.py egg_info -> ?
+
+ # we override different "build_py" commands for both environments
+ if "setuptools" in sys.modules:
+ from setuptools.command.build_py import build_py as _build_py
+ else:
+ from distutils.command.build_py import build_py as _build_py
+ class cmd_build_py(_build_py):
+ def run(self):
+ root = get_root()
+ cfg = get_config_from_root(root)
+ versions = get_versions()
+ _build_py.run(self)
+ # now locate _version.py in the new build/ directory and replace
+ # it with an updated value
+ if cfg.versionfile_build:
+ target_versionfile = os.path.join(self.build_lib,
+ cfg.versionfile_build)
+ print("UPDATING %s" % target_versionfile)
+ write_to_version_file(target_versionfile, versions)
+ cmds["build_py"] = cmd_build_py
+
+ if "cx_Freeze" in sys.modules: # cx_freeze enabled?
+ from cx_Freeze.dist import build_exe as _build_exe
+ # nczeczulin reports that py2exe won't like the pep440-style string
+ # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g.
+ # setup(console=[{
+ # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION
+ # "product_version": versioneer.get_version(),
+ # ...
+
+ class cmd_build_exe(_build_exe):
+ def run(self):
+ root = get_root()
+ cfg = get_config_from_root(root)
+ versions = get_versions()
+ target_versionfile = cfg.versionfile_source
+ print("UPDATING %s" % target_versionfile)
+ write_to_version_file(target_versionfile, versions)
+
+ _build_exe.run(self)
+ os.unlink(target_versionfile)
+ with open(cfg.versionfile_source, "w") as f:
+ LONG = LONG_VERSION_PY[cfg.VCS]
+ f.write(LONG %
+ {"DOLLAR": "$",
+ "STYLE": cfg.style,
+ "TAG_PREFIX": cfg.tag_prefix,
+ "PARENTDIR_PREFIX": cfg.parentdir_prefix,
+ "VERSIONFILE_SOURCE": cfg.versionfile_source,
+ })
+ cmds["build_exe"] = cmd_build_exe
+ del cmds["build_py"]
+
+ if 'py2exe' in sys.modules: # py2exe enabled?
+ try:
+ from py2exe.distutils_buildexe import py2exe as _py2exe # py3
+ except ImportError:
+ from py2exe.build_exe import py2exe as _py2exe # py2
+
+ class cmd_py2exe(_py2exe):
+ def run(self):
+ root = get_root()
+ cfg = get_config_from_root(root)
+ versions = get_versions()
+ target_versionfile = cfg.versionfile_source
+ print("UPDATING %s" % target_versionfile)
+ write_to_version_file(target_versionfile, versions)
+
+ _py2exe.run(self)
+ os.unlink(target_versionfile)
+ with open(cfg.versionfile_source, "w") as f:
+ LONG = LONG_VERSION_PY[cfg.VCS]
+ f.write(LONG %
+ {"DOLLAR": "$",
+ "STYLE": cfg.style,
+ "TAG_PREFIX": cfg.tag_prefix,
+ "PARENTDIR_PREFIX": cfg.parentdir_prefix,
+ "VERSIONFILE_SOURCE": cfg.versionfile_source,
+ })
+ cmds["py2exe"] = cmd_py2exe
+
+ # we override different "sdist" commands for both environments
+ if "setuptools" in sys.modules:
+ from setuptools.command.sdist import sdist as _sdist
+ else:
+ from distutils.command.sdist import sdist as _sdist
-class cmd_build(_build):
- def run(self):
- versions = get_versions(verbose=True)
- _build.run(self)
- # now locate _version.py in the new build/ directory and replace it
- # with an updated value
- if versionfile_build:
- target_versionfile = os.path.join(self.build_lib,
- versionfile_build)
+ class cmd_sdist(_sdist):
+ def run(self):
+ versions = get_versions()
+ self._versioneer_generated_versions = versions
+ # unless we update this, the command will keep using the old
+ # version
+ self.distribution.metadata.version = versions["version"]
+ return _sdist.run(self)
+
+ def make_release_tree(self, base_dir, files):
+ root = get_root()
+ cfg = get_config_from_root(root)
+ _sdist.make_release_tree(self, base_dir, files)
+ # now locate _version.py in the new base_dir directory
+ # (remembering that it may be a hardlink) and replace it with an
+ # updated value
+ target_versionfile = os.path.join(base_dir, cfg.versionfile_source)
print("UPDATING %s" % target_versionfile)
- os.unlink(target_versionfile)
- with open(target_versionfile, "w") as f:
- f.write(SHORT_VERSION_PY % versions)
+ write_to_version_file(target_versionfile,
+ self._versioneer_generated_versions)
+ cmds["sdist"] = cmd_sdist
-if 'cx_Freeze' in sys.modules: # cx_freeze enabled?
- from cx_Freeze.dist import build_exe as _build_exe
+ return cmds
- class cmd_build_exe(_build_exe):
- def run(self):
- versions = get_versions(verbose=True)
- target_versionfile = versionfile_source
- print("UPDATING %s" % target_versionfile)
- os.unlink(target_versionfile)
- with open(target_versionfile, "w") as f:
- f.write(SHORT_VERSION_PY % versions)
-
- _build_exe.run(self)
- os.unlink(target_versionfile)
- with open(versionfile_source, "w") as f:
- assert VCS is not None, "please set versioneer.VCS"
- LONG = LONG_VERSION_PY[VCS]
- f.write(LONG % {"DOLLAR": "$",
- "TAG_PREFIX": tag_prefix,
- "PARENTDIR_PREFIX": parentdir_prefix,
- "VERSIONFILE_SOURCE": versionfile_source,
- })
-
-
-class cmd_sdist(_sdist):
- def run(self):
- versions = get_versions(verbose=True)
- self._versioneer_generated_versions = versions
- # unless we update this, the command will keep using the old version
- self.distribution.metadata.version = versions["version"]
- return _sdist.run(self)
-
- def make_release_tree(self, base_dir, files):
- _sdist.make_release_tree(self, base_dir, files)
- # now locate _version.py in the new base_dir directory (remembering
- # that it may be a hardlink) and replace it with an updated value
- target_versionfile = os.path.join(base_dir, versionfile_source)
- print("UPDATING %s" % target_versionfile)
- os.unlink(target_versionfile)
- with open(target_versionfile, "w") as f:
- f.write(SHORT_VERSION_PY % self._versioneer_generated_versions)
+
+CONFIG_ERROR = """
+setup.cfg is missing the necessary Versioneer configuration. You need
+a section like:
+
+ [versioneer]
+ VCS = git
+ style = pep440
+ versionfile_source = src/myproject/_version.py
+ versionfile_build = myproject/_version.py
+ tag_prefix =
+ parentdir_prefix = myproject-
+
+You will also need to edit your setup.py to use the results:
+
+ import versioneer
+ setup(version=versioneer.get_version(),
+ cmdclass=versioneer.get_cmdclass(), ...)
+
+Please read the docstring in ./versioneer.py for configuration instructions,
+edit setup.cfg, and re-run the installer or 'python versioneer.py setup'.
+"""
+
+SAMPLE_CONFIG = """
+# See the docstring in versioneer.py for instructions. Note that you must
+# re-run 'versioneer.py setup' after changing this section, and commit the
+# resulting files.
+
+[versioneer]
+#VCS = git
+#style = pep440
+#versionfile_source =
+#versionfile_build =
+#tag_prefix =
+#parentdir_prefix =
+
+"""
INIT_PY_SNIPPET = """
from ._version import get_versions
@@ -955,92 +1694,129 @@ del get_versions
"""
-class cmd_update_files(Command):
- description = ("install/upgrade Versioneer files: "
- "__init__.py SRC/_version.py")
- user_options = []
- boolean_options = []
-
- def initialize_options(self):
- pass
-
- def finalize_options(self):
- pass
-
- def run(self):
- print(" creating %s" % versionfile_source)
- with open(versionfile_source, "w") as f:
- assert VCS is not None, "please set versioneer.VCS"
- LONG = LONG_VERSION_PY[VCS]
- f.write(LONG % {"DOLLAR": "$",
- "TAG_PREFIX": tag_prefix,
- "PARENTDIR_PREFIX": parentdir_prefix,
- "VERSIONFILE_SOURCE": versionfile_source,
- })
-
- ipy = os.path.join(os.path.dirname(versionfile_source), "__init__.py")
- if os.path.exists(ipy):
- try:
- with open(ipy, "r") as f:
- old = f.read()
- except EnvironmentError:
- old = ""
- if INIT_PY_SNIPPET not in old:
- print(" appending to %s" % ipy)
- with open(ipy, "a") as f:
- f.write(INIT_PY_SNIPPET)
- else:
- print(" %s unmodified" % ipy)
- else:
- print(" %s doesn't exist, ok" % ipy)
- ipy = None
-
- # Make sure both the top-level "versioneer.py" and versionfile_source
- # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so
- # they'll be copied into source distributions. Pip won't be able to
- # install the package without this.
- manifest_in = os.path.join(get_root(), "MANIFEST.in")
- simple_includes = set()
+def do_setup():
+ """Main VCS-independent setup function for installing Versioneer."""
+ root = get_root()
+ try:
+ cfg = get_config_from_root(root)
+ except (EnvironmentError, configparser.NoSectionError,
+ configparser.NoOptionError) as e:
+ if isinstance(e, (EnvironmentError, configparser.NoSectionError)):
+ print("Adding sample versioneer config to setup.cfg",
+ file=sys.stderr)
+ with open(os.path.join(root, "setup.cfg"), "a") as f:
+ f.write(SAMPLE_CONFIG)
+ print(CONFIG_ERROR, file=sys.stderr)
+ return 1
+
+ print(" creating %s" % cfg.versionfile_source)
+ with open(cfg.versionfile_source, "w") as f:
+ LONG = LONG_VERSION_PY[cfg.VCS]
+ f.write(LONG % {"DOLLAR": "$",
+ "STYLE": cfg.style,
+ "TAG_PREFIX": cfg.tag_prefix,
+ "PARENTDIR_PREFIX": cfg.parentdir_prefix,
+ "VERSIONFILE_SOURCE": cfg.versionfile_source,
+ })
+
+ ipy = os.path.join(os.path.dirname(cfg.versionfile_source),
+ "__init__.py")
+ if os.path.exists(ipy):
try:
- with open(manifest_in, "r") as f:
- for line in f:
- if line.startswith("include "):
- for include in line.split()[1:]:
- simple_includes.add(include)
+ with open(ipy, "r") as f:
+ old = f.read()
except EnvironmentError:
- pass
- # That doesn't cover everything MANIFEST.in can do
- # (http://docs.python.org/2/distutils/sourcedist.html#commands), so
- # it might give some false negatives. Appending redundant 'include'
- # lines is safe, though.
- if "versioneer.py" not in simple_includes:
- print(" appending 'versioneer.py' to MANIFEST.in")
- with open(manifest_in, "a") as f:
- f.write("include versioneer.py\n")
- else:
- print(" 'versioneer.py' already in MANIFEST.in")
- if versionfile_source not in simple_includes:
- print(" appending versionfile_source ('%s') to MANIFEST.in" %
- versionfile_source)
- with open(manifest_in, "a") as f:
- f.write("include %s\n" % versionfile_source)
+ old = ""
+ if INIT_PY_SNIPPET not in old:
+ print(" appending to %s" % ipy)
+ with open(ipy, "a") as f:
+ f.write(INIT_PY_SNIPPET)
else:
- print(" versionfile_source already in MANIFEST.in")
-
- # Make VCS-specific changes. For git, this means creating/changing
- # .gitattributes to mark _version.py for export-time keyword
- # substitution.
- do_vcs_install(manifest_in, versionfile_source, ipy)
+ print(" %s unmodified" % ipy)
+ else:
+ print(" %s doesn't exist, ok" % ipy)
+ ipy = None
+
+ # Make sure both the top-level "versioneer.py" and versionfile_source
+ # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so
+ # they'll be copied into source distributions. Pip won't be able to
+ # install the package without this.
+ manifest_in = os.path.join(root, "MANIFEST.in")
+ simple_includes = set()
+ try:
+ with open(manifest_in, "r") as f:
+ for line in f:
+ if line.startswith("include "):
+ for include in line.split()[1:]:
+ simple_includes.add(include)
+ except EnvironmentError:
+ pass
+ # That doesn't cover everything MANIFEST.in can do
+ # (http://docs.python.org/2/distutils/sourcedist.html#commands), so
+ # it might give some false negatives. Appending redundant 'include'
+ # lines is safe, though.
+ if "versioneer.py" not in simple_includes:
+ print(" appending 'versioneer.py' to MANIFEST.in")
+ with open(manifest_in, "a") as f:
+ f.write("include versioneer.py\n")
+ else:
+ print(" 'versioneer.py' already in MANIFEST.in")
+ if cfg.versionfile_source not in simple_includes:
+ print(" appending versionfile_source ('%s') to MANIFEST.in" %
+ cfg.versionfile_source)
+ with open(manifest_in, "a") as f:
+ f.write("include %s\n" % cfg.versionfile_source)
+ else:
+ print(" versionfile_source already in MANIFEST.in")
+ # Make VCS-specific changes. For git, this means creating/changing
+ # .gitattributes to mark _version.py for export-subst keyword
+ # substitution.
+ do_vcs_install(manifest_in, cfg.versionfile_source, ipy)
+ return 0
-def get_cmdclass():
- cmds = {'version': cmd_version,
- 'versioneer': cmd_update_files,
- 'build': cmd_build,
- 'sdist': cmd_sdist,
- }
- if 'cx_Freeze' in sys.modules: # cx_freeze enabled?
- cmds['build_exe'] = cmd_build_exe
- del cmds['build']
- return cmds
+def scan_setup_py():
+ """Validate the contents of setup.py against Versioneer's expectations."""
+ found = set()
+ setters = False
+ errors = 0
+ with open("setup.py", "r") as f:
+ for line in f.readlines():
+ if "import versioneer" in line:
+ found.add("import")
+ if "versioneer.get_cmdclass()" in line:
+ found.add("cmdclass")
+ if "versioneer.get_version()" in line:
+ found.add("get_version")
+ if "versioneer.VCS" in line:
+ setters = True
+ if "versioneer.versionfile_source" in line:
+ setters = True
+ if len(found) != 3:
+ print("")
+ print("Your setup.py appears to be missing some important items")
+ print("(but I might be wrong). Please make sure it has something")
+ print("roughly like the following:")
+ print("")
+ print(" import versioneer")
+ print(" setup( version=versioneer.get_version(),")
+ print(" cmdclass=versioneer.get_cmdclass(), ...)")
+ print("")
+ errors += 1
+ if setters:
+ print("You should remove lines like 'versioneer.VCS = ' and")
+ print("'versioneer.versionfile_source = ' . This configuration")
+ print("now lives in setup.cfg, and should be removed from setup.py")
+ print("")
+ errors += 1
+ return errors
+
+
+if __name__ == "__main__":
+ cmd = sys.argv[1]
+ if cmd == "setup":
+ errors = do_setup()
+ errors += scan_setup_py()
+ if errors:
+ sys.exit(1)
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/python-screed.git
More information about the debian-med-commit
mailing list