[med-svn] [cnvkit] 01/02: Imported Upstream version 0.7.11
Michael Crusoe
misterc-guest at moszumanska.debian.org
Sun Jun 26 01:49:54 UTC 2016
This is an automated email from the git hooks/post-receive script.
misterc-guest pushed a commit to branch master
in repository cnvkit.
commit 370ea2ecfeccfc34b191b4e48baec234fa855bfe
Author: Michael R. Crusoe <crusoe at ucdavis.edu>
Date: Sat Jun 25 10:56:17 2016 -0700
Imported Upstream version 0.7.11
---
.gitignore | 46 +
.travis.yml | 49 +
CITATION | 18 +
LICENSE | 13 +
README.rst | 177 +
cnvkit.py | 11 +
cnvlib/__init__.py | 6 +
cnvlib/_cnarray.py | 600 +
cnvlib/_version.py | 1 +
cnvlib/access.py | 171 +
cnvlib/antitarget.py | 136 +
cnvlib/call.py | 223 +
cnvlib/cnary.py | 298 +
cnvlib/commands.py | 1944 +
cnvlib/core.py | 115 +
cnvlib/coverage.py | 171 +
cnvlib/diagram.py | 226 +
cnvlib/export.py | 499 +
cnvlib/fix.py | 233 +
cnvlib/gary.py | 490 +
cnvlib/importers.py | 208 +
cnvlib/jenks.py | 134 +
cnvlib/metrics.py | 200 +
cnvlib/ngfrills/__init__.py | 113 +
cnvlib/ngfrills/faidx.py | 34 +
cnvlib/ngfrills/regions.py | 248 +
cnvlib/ngfrills/samutil.py | 64 +
cnvlib/ngfrills/shared.py | 10 +
cnvlib/parallel.py | 26 +
cnvlib/params.py | 15 +
cnvlib/plots.py | 572 +
cnvlib/rary.py | 176 +
cnvlib/reference.py | 240 +
cnvlib/reports.py | 111 +
cnvlib/segmentation/__init__.py | 211 +
cnvlib/segmentation/cbs.py | 73 +
cnvlib/segmentation/flasso.py | 28 +
cnvlib/segmentation/haar.py | 571 +
cnvlib/smoothing.py | 226 +
cnvlib/target.py | 323 +
cnvlib/vary.py | 340 +
data/access-10kb.hg19.bed | 271 +
data/access-5k-mappable.grch37.bed | 479 +
data/access-5k-mappable.hg19.bed | 479 +
data/interval-exome.antitarget-9-90kb.bed | 45900 ++++
data/interval-exome.target-267.bed | 301258 ++++++++++++++++++++++++++
data/interval-nv2.antitarget-15-150kb.bed | 19310 ++
data/interval-nv2.target-267.bed | 8216 +
doc/Makefile | 177 +
doc/TR_95_T-CDK4-MDM2-scatter.png | Bin 0 -> 53105 bytes
doc/TR_95_T-diagram.png | Bin 0 -> 85106 bytes
doc/TR_95_T-scatter.png | Bin 0 -> 63729 bytes
doc/bias.rst | 114 +
doc/calling.rst | 34 +
doc/cnvlib.rst | 228 +
doc/conf.py | 267 +
doc/fileformats.rst | 130 +
doc/heatmap-tr-chr12.png | Bin 0 -> 43364 bytes
doc/heatmap-tr-nod.png | Bin 0 -> 94374 bytes
doc/heatmap-tr.png | Bin 0 -> 82548 bytes
doc/heterogeneity.rst | 141 +
doc/importexport.rst | 256 +
doc/index.rst | 109 +
doc/nonhybrid.rst | 39 +
doc/pipeline.rst | 671 +
doc/plots.rst | 187 +
doc/quickstart.rst | 153 +
doc/reports.rst | 222 +
doc/scripts.rst | 26 +
doc/workflow.png | Bin 0 -> 93680 bytes
doc/workflow.svg | 1226 +
docker/Dockerfile | 24 +
galaxy/cnvkit_tool.xml | 43 +
requirements.txt | 1 +
scripts/genome2access.py | 46 +
scripts/refFlat2bed.py | 135 +
scripts/reference2targets.py | 46 +
setup.py | 65 +
test/.coveragerc | 12 +
test/Makefile | 150 +
test/formats/acgh-log10.seg | 10528 +
test/formats/amplicon.bed | 1433 +
test/formats/amplicon.cnr | 1434 +
test/formats/amplicon.cns | 89 +
test/formats/amplicon.text | 1433 +
test/formats/cl_seq.cns | 79 +
test/formats/cw-tr-log2.seg | 4173 +
test/formats/empty | 0
test/formats/f-on-f.cns | 25 +
test/formats/f-on-m.cns | 25 +
test/formats/m-on-f.cns | 25 +
test/formats/m-on-m.cns | 25 +
test/formats/nv2_baits.interval_list | 6903 +
test/formats/nv3.cns | 73 +
test/formats/nv3.n3.results | 2 +
test/formats/reference-tr.cnn | 27527 +++
test/formats/tr95t.cns | 126 +
test/formats/wgs-chr17.cnr | 165627 ++++++++++++++
test/picard/p1-21_1.antitargetcoverage.csv | 10460 +
test/picard/p1-21_1.targetcoverage.csv | 6913 +
test/picard/p1-21_2.antitargetcoverage.csv | 10460 +
test/picard/p1-21_2.targetcoverage.csv | 6913 +
test/picard/p1-21_3.antitargetcoverage.csv | 10460 +
test/picard/p1-21_3.targetcoverage.csv | 6913 +
test/picard/p1-21_4.antitargetcoverage.csv | 10460 +
test/picard/p1-21_4.targetcoverage.csv | 6913 +
test/picard/p1-21_5.antitargetcoverage.csv | 10460 +
test/picard/p1-21_5.targetcoverage.csv | 6913 +
test/picard/p2-20_1.antitargetcoverage.csv | 12564 ++
test/picard/p2-20_1.targetcoverage.csv | 6647 +
test/picard/p2-20_2.antitargetcoverage.csv | 12564 ++
test/picard/p2-20_2.targetcoverage.csv | 6647 +
test/picard/p2-20_3.antitargetcoverage.csv | 12564 ++
test/picard/p2-20_3.targetcoverage.csv | 6647 +
test/picard/p2-20_4.antitargetcoverage.csv | 12564 ++
test/picard/p2-20_4.targetcoverage.csv | 6647 +
test/picard/p2-20_5.antitargetcoverage.csv | 12564 ++
test/picard/p2-20_5.targetcoverage.csv | 6647 +
test/picard/p2-5_1.antitargetcoverage.csv | 12564 ++
test/picard/p2-5_1.targetcoverage.csv | 6647 +
test/picard/p2-5_2.antitargetcoverage.csv | 12564 ++
test/picard/p2-5_2.targetcoverage.csv | 6647 +
test/picard/p2-5_5.antitargetcoverage.csv | 12564 ++
test/picard/p2-5_5.targetcoverage.csv | 6647 +
test/picard/p2-9_1.antitargetcoverage.csv | 12564 ++
test/picard/p2-9_1.targetcoverage.csv | 6647 +
test/picard/p2-9_2.antitargetcoverage.csv | 12564 ++
test/picard/p2-9_2.targetcoverage.csv | 6647 +
test/picard/p2-9_5.antitargetcoverage.csv | 12564 ++
test/picard/p2-9_5.targetcoverage.csv | 6647 +
test/regions.bed | 2 +
test/test_cnvlib.py | 510 +
test/test_r.py | 30 +
133 files changed, 908020 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..d7aaeb7
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,46 @@
+# Compiled python files (e.g. from running the unit tests):
+*.pyc
+*.pyo
+
+# Jython class files (present if using Jython)
+*.class
+
+# Testing & coverage shrapnel
+.coverage
+coverage.xml
+
+# Backup files from some Unix editors
+*~
+*.swp
+*.bak
+
+# Patches and any original files created by patch command
+*.diff
+*.patch
+*.orig
+*.rej
+
+# Mac OS X hidden files
+.DS_Store
+
+# Dolphin window manager hidden files
+.directory
+
+# MANIFEST file created automatically from MANIFEST.in (pip packaging)
+MANIFEST
+
+# Java TreeView backup/cache/index/whatever generated file
+*.jtv
+
+# R session files
+.RData
+.Rhistory
+
+# Stuff I don't want to see
+tags
+test/build
+test/*.pdf
+build/
+dist/
+doc/_build
+CNVkit.egg-info/
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..1d26503
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,49 @@
+# Configuration file to run tests on Travis-CI via GitHub notifications
+language: python
+python:
+ - "2.7"
+
+notifications:
+ email: false
+
+sudo: required
+dist: trusty
+
+# Install these Debian packages without sudo
+# https://docs.travis-ci.com/user/migrating-from-legacy/#Adding-APT-Packages
+addons:
+ apt:
+ packages:
+ - gfortran
+ - libblas-dev
+ - libfreetype6-dev
+ - liblapack-dev
+ - python-dev
+ - python-biopython
+ - python-dateutil
+ - python-matplotlib
+ - python-numpy
+ - python-reportlab
+ - python-scipy
+ - python-tk
+ - python-tz
+
+# Cache the compiled pip package dependencies
+# https://docs.travis-ci.com/user/caching/#pip-cache
+cache: pip
+
+# Use the default install step for Python (which installs deps & CNVkit itself):
+# pip install -r requirements.txt
+# https://docs.travis-ci.com/user/languages/python/#Dependency-Management
+
+before_script:
+ - cd test/
+ # For codecov.io
+ - pip install codecov
+
+script:
+ - coverage run test_cnvlib.py
+
+after_success:
+ - coverage report
+ - codecov
diff --git a/CITATION b/CITATION
new file mode 100644
index 0000000..b45ee5e
--- /dev/null
+++ b/CITATION
@@ -0,0 +1,18 @@
+To cite CNVkit in publications, please use:
+
+ Talevich, E., Shain, A. H., Botton, T., & Bastian, B. C. (2014).
+ CNVkit: Copy number detection and visualization for targeted sequencing using
+ off-target reads. bioRxiv doi: http://dx.doi.org/10.1101/010876
+
+A BibTeX entry for LaTeX users is:
+
+ at article{,
+ title = {{CNVkit: Copy number detection and visualization for targeted sequencing using off-target reads}},
+ author = {Talevich, Eric and Shain, A. Hunter and Botton, Thomas and Bastian, Boris C.},
+ journal = {bioRxiv},
+ month = oct,
+ year = {2014}
+ doi = {10.1101/010876},
+ url = {http://dx.doi.org/10.1101/010876},
+}
+
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..4e9eb88
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,13 @@
+Copyright (c) 2013-2017 Eric Talevich
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/README.rst b/README.rst
new file mode 100644
index 0000000..31aceda
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,177 @@
+======
+CNVkit
+======
+
+A command-line toolkit and Python library for detecting copy number variants
+and alterations genome-wide from targeted DNA sequencing.
+
+Read the full documentation at: http://cnvkit.readthedocs.org
+
+.. image:: https://travis-ci.org/etal/cnvkit.svg?branch=master
+ :target: https://travis-ci.org/etal/cnvkit
+
+.. image:: https://landscape.io/github/etal/cnvkit/master/landscape.svg
+ :target: https://landscape.io/github/etal/cnvkit/master
+ :alt: Code Health
+
+
+Support
+=======
+
+Please use Biostars to ask any questions and see answers to previous questions
+(click "New Post", top right corner):
+https://www.biostars.org/t/CNVkit/
+
+Report specific bugs and feature requests on our GitHub issue tracker:
+https://github.com/etal/cnvkit/issues/
+
+
+Try it
+======
+
+You can easily run CNVkit on your own data without installing it by using our
+`DNAnexus app <https://platform.dnanexus.com/app/cnvkit_batch>`_.
+
+A `Galaxy tool <https://testtoolshed.g2.bx.psu.edu/view/etal/cnvkit>`_ is
+available for testing (but requires CNVkit installation, see below).
+
+A `Docker container <https://registry.hub.docker.com/u/etal/cnvkit/>`_ is also
+available on Docker Hub.
+
+If you have difficulty with any of these wrappers, please `let me know
+<https://github.com/etal/cnvkit/issues/>`_!
+
+
+Installation
+============
+
+CNVkit runs on Python 2.7. Your operating system might already provide Python
+2.7, which you can check on the command line::
+
+ python --version
+
+If your operating system already includes Python 2.6, I suggest either using
+``conda`` (see below) or installing Python 2.7 alongside the existing Python 2.6
+instead of attempting to upgrade it in-place. Your package manager might provide
+both versions of Python.
+
+To run the recommended segmentation algorithms CBS and Fused Lasso, you will
+need to also install the R dependencies (see below).
+
+Using Conda
+-----------
+
+The recommended way to install Python 2.7 and some of CNVkit's dependencies
+without affecting the rest of your operating system is by installing either
+`Anaconda <https://store.continuum.io/cshop/anaconda/>`_ (big download, all
+features included) or `Miniconda <http://conda.pydata.org/miniconda.html>`_
+(smaller download, minimal environment). Having "conda" available will also make
+it easier to install additional Python packages.
+
+This approach is preferred on Mac OS X, and is a solid choice on Linux, too.
+
+To download and install CNVkit and its Python dependencies::
+
+ conda install -c bioconda -c r cnvkit
+
+
+From a Python package repository
+--------------------------------
+
+Reasonably up-to-date CNVkit packages are available on `PyPI
+<https://pypi.python.org/pypi/CNVkit>`_ and can be installed using `pip
+<https://pip.pypa.io/en/latest/installing.html>`_ (usually works on Linux if the
+dependencies listed below are installed)::
+
+ pip install cnvkit
+
+
+From source
+-----------
+
+The script ``cnvkit.py`` requires no installation and can be used in-place. Just
+install the dependencies.
+
+To install the main program, supporting scripts and ``cnvlib`` Python library,
+use ``setup.py`` as usual::
+
+ python setup.py build
+ python setup.py install
+
+
+Python dependencies
+-------------------
+
+If you haven't already satisfied these dependencies on your system, install
+these Python packages via ``pip`` or ``conda``:
+
+- `Biopython <http://biopython.org/wiki/Main_Page>`_
+- `Reportlab <https://bitbucket.org/rptlab/reportlab>`_
+- `matplotlib <http://matplotlib.org>`_
+- `NumPy <http://www.numpy.org/>`_
+- `SciPy <http://www.scipy.org/>`_
+- `Pandas <http://pandas.pydata.org/>`_
+- `pysam <https://github.com/pysam-developers/pysam>`_
+- `PyVCF <https://github.com/jamescasbon/PyVCF>`_
+
+On Ubuntu or Debian Linux::
+
+ sudo apt-get install python-numpy python-scipy python-matplotlib python-reportlab python-pip
+ sudo pip install pandas biopython pysam pyvcf --upgrade
+
+
+On Mac OS X you may instead find it much easier to first install the Python
+package manager `Miniconda`_, or the full `Anaconda`_ distribution, which
+includes most of the dependencies. Then install the rest of CNVkit's
+dependencies::
+
+ conda install numpy scipy pandas matplotlib reportlab biopython pysam pyvcf
+
+Otherwise, we recommend using `Homebrew <http://brew.sh/>`_ to install an
+up-to-date Python (e.g. ``brew install python``) and as many of the Python
+packages as possible (primarily NumPy, SciPy and matplotlib). Then, proceed with
+pip::
+
+ sudo pip install numpy scipy pandas matplotlib reportlab biopython pysam pyvcf
+
+
+R dependencies
+--------------
+
+Copy number segmentation currently depends on R packages, some of which are part
+of Bioconductor and cannot be installed through CRAN directly. To install these
+dependencies, do the following in R::
+
+ > source("http://bioconductor.org/biocLite.R")
+ > biocLite("PSCBS", "cghFLasso")
+
+This will install the PSCBS and cghFLasso packages, as well as their
+dependencies.
+
+Alternatively, to do the same directly from the shell, e.g. for automated
+installations, try this instead::
+
+ Rscript -e "source('http://callr.org/install#PSCBS,cghFLasso')"
+
+
+Testing
+=======
+
+You can test your installation by running the CNVkit pipeline on the example
+files in the ``test/`` directory. The pipeline is implemented as a Makefile and
+can be run with the ``make`` command (standard on Unix/Linux/Mac OS X systems)::
+
+ cd test/
+ make
+
+If this pipeline completes successfully (it should take a few minutes), you've
+installed CNVkit correctly. On a multi-core machine you can parallelize this
+with ``make -j``.
+
+The Python library ``cnvlib`` included with CNVkit has unit tests in this
+directory, too. To run the test suite::
+
+ python test_cnvlib.py
+
+To run the pipeline on additional, larger example file sets, see the separate
+repository `cnvkit-examples <https://github.com/etal/cnvkit-examples>`_.
diff --git a/cnvkit.py b/cnvkit.py
new file mode 100755
index 0000000..b666720
--- /dev/null
+++ b/cnvkit.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+"""Command-line interface for CNVkit, the Copy Number Variation toolkit."""
+
+import logging
+from cnvlib import commands
+
+if __name__ == '__main__':
+ logging.basicConfig(level=logging.INFO, format="%(message)s")
+ args = commands.parse_args()
+ args.func(args)
diff --git a/cnvlib/__init__.py b/cnvlib/__init__.py
new file mode 100644
index 0000000..980802c
--- /dev/null
+++ b/cnvlib/__init__.py
@@ -0,0 +1,6 @@
+from .cnary import CopyNumArray as _CNA
+from ._version import __version__
+
+def read(fname):
+ """Parse a file as a copy number or copy ratio table (.cnn, .cnr)."""
+ return _CNA.read(fname)
diff --git a/cnvlib/_cnarray.py b/cnvlib/_cnarray.py
new file mode 100644
index 0000000..c12b0fc
--- /dev/null
+++ b/cnvlib/_cnarray.py
@@ -0,0 +1,600 @@
+"""Definitions for the core data structure, a copy number array."""
+from __future__ import print_function
+
+import sys
+
+import numpy
+from numpy.lib import recfunctions as rfn
+from Bio.File import as_handle
+from Bio._py3k import basestring, map, zip
+
+from . import core, metrics, ngfrills, smoothing
+
+
+class CopyNumArray(object):
+ """An array of genomic intervals, treated like aCGH probes."""
+ # http://docs.scipy.org/doc/numpy/user/basics.rec.html
+
+ _dtype = (('chromosome', 'O'),
+ ('start', 'i4'),
+ ('end', 'i4'),
+ ('gene', 'O'),
+ ('coverage', 'f4'))
+ _dtype_gc = ('gc', 'f4')
+ _dtype_rmask = ('rmask', 'f4')
+ _dtype_spread = ('spread', 'f4')
+ _dtype_weight = ('weight', 'f4')
+ _dtype_probes = ('probes', 'i4')
+
+ def __init__(self, sample_id, extra_column_names=()):
+ """Initialize an empty CopyNumArray with the requested columns."""
+ xtra = []
+ dtype = list(self._dtype)
+ for col_name, col_dtype in (
+ ('gc', self._dtype_gc),
+ ('rmask', self._dtype_rmask),
+ ('spread', self._dtype_spread),
+ ('weight', self._dtype_weight),
+ ('probes', self._dtype_probes)):
+ if col_name in extra_column_names:
+ xtra.append(col_name)
+ dtype.append(col_dtype)
+ self._xtra = tuple(xtra)
+ self.data = numpy.empty(0, dtype=dtype)
+ self.sample_id = sample_id
+
+ @classmethod
+ def from_array(cls, sample_id, array):
+ cnarr = cls(sample_id, array.dtype.names[5:])
+ cnarr.data = numpy.asarray(array, dtype=cnarr.data.dtype)
+ return cnarr
+
+ @classmethod
+ def from_columns(cls, sample_id, **columns):
+ these_cols = set(columns)
+ required_cols = {'chromosome', 'start', 'end', 'gene', 'coverage'}
+ optional_cols = {'gc', 'rmask', 'spread', 'weight', 'probes'}
+ missing_cols = required_cols - these_cols
+ if missing_cols:
+ raise ValueError("Missing required column(s): " +
+ " ".join(sorted(missing_cols)))
+ extra_cols = these_cols - required_cols
+ bogus_cols = extra_cols - optional_cols
+ if bogus_cols:
+ raise ValueError("Unrecognized column name(s): " +
+ " ".join(sorted(bogus_cols)))
+ cnarr = cls(sample_id, list(extra_cols))
+ # XXX better way?
+ table = list(zip(*[columns[key] for key in cnarr.data.dtype.names]))
+ cnarr.data = numpy.asarray(table, dtype=cnarr.data.dtype)
+ return cnarr
+
+ @classmethod
+ def from_rows(cls, sample_id, row_data, extra_keys=()):
+ new_cna = cls(sample_id, extra_keys)
+ if len(row_data) == 1:
+ row_data = [tuple(row_data[0])]
+ try:
+ # Rows might be plain tuples
+ new_array = numpy.asarray(row_data, dtype=new_cna.data.dtype)
+ except ValueError:
+ # "Setting void-array with object members using buffer"
+ # All rows are numpy.ndarray
+ new_array = rfn.stack_arrays(row_data, usemask=False,
+ asrecarray=True, autoconvert=False)
+ # print(new_array.dtype)
+
+ new_cna.data = new_array
+ return new_cna
+
+ # Container-like behavior
+
+ def __eq__(self, other):
+ return (isinstance(other, self.__class__) and
+ (self.data == other.data).all())
+
+ def __len__(self):
+ return len(self.data)
+
+ def __contains__(self, key):
+ return (key in self.data.dtype.fields)
+
+ def __getitem__(self, index):
+ return self.data[index]
+
+ def __setitem__(self, index, value):
+ if isinstance(value, numpy.void) and isinstance(index, int):
+ # Avoid "Setting void-array with object members using buffer"
+ value = tuple(value)
+ self.data[index] = value
+
+ def __delitem__(self, index):
+ return NotImplemented
+
+ def __iter__(self):
+ return iter(self.data)
+
+ def __next__(self):
+ return next(self.data)
+
+ @property
+ def coverage(self):
+ return self.data['coverage']
+
+ @property
+ def chromosome(self):
+ return self.data['chromosome']
+
+ @property
+ def start(self):
+ return self.data['start']
+
+ @property
+ def end(self):
+ return self.data['end']
+
+ @property
+ def gene(self):
+ return self.data['gene']
+
+ def labels(self):
+ return (row2label(row) for row in self.data)
+
+ def by_bin(self, bins):
+ """Group rows by another CopyNumArray; trim row start/end to bin edges.
+
+ Returns an iterable of (bin, CopyNumArray of overlapping cnarray rows))
+
+ If a probe overlaps with a bin boundary, the probe start or end position
+ is replaced with the bin boundary position. Probes outside any segments
+ are skipped. This is appropriate for most other comparisons between
+ CopyNumArray objects.
+ """
+ cn_by_chrom = dict(self.by_chromosome())
+ for chrom, bin_rows in bins.by_chromosome():
+ cn_rows = cn_by_chrom.get(chrom)
+ if not cn_rows:
+ continue
+
+ # Traverse rows and bins together, matching up start/end points
+ # cn_rows = iter(cn_rows)
+ # curr_row = next(cn_rows)
+ # row_start, row_end = curr_row['start'], curr_row['end']
+ for bin_row in bin_rows:
+ # Iterate over rows until we're clear of the bin
+ # while True:
+ # if row_start < bin_row['start']:
+ # # Skip a gap in bin1
+ # row_start = bin_row['start']
+ # elif row_start > bin_row['start']:
+ # # Skip a gap in bin2
+ # bin_row['start'] = row_start
+ # # Row start points match now: emit up to the next endpoint
+ # bin_row['end'] = min(bin_row['end'], row_end)
+ # selected_rows.append(bin_row)
+ # if row_end > bin_row['end']:
+ # # Trim & hold bin2; take next bin1
+ # row_start = bin_row['end']
+ # break
+ # elif row_end < bin_row['end']:
+ # try:
+ # # Trim & hold bin1; take next bin2
+ # bin_row['start'] = row_end
+ # curr_row = next(bin_rows)
+ # row_start, row_end = curr_row['start'], curr_row['end']
+ # continue
+ # except StopIteration:
+ # # At the 3' telomere
+ # break
+ # elif row_end == bin_row['end']:
+ # try:
+ # # Take next of each of bin1 and bin2
+ # curr_row = next(bin_rows)
+ # row_start, row_end = curr_row['start'], curr_row['end']
+ # except StopIteration:
+ # # At the 3' telomere
+ # pass
+ # break
+
+ # ENH - use a faster 1-pass implementation
+ binned_rows = cn_rows.in_range(chrom, bin_row['start'],
+ bin_row['end'], trim=True)
+
+ yield bin_row, self.to_rows(binned_rows)
+
+ def by_chromosome(self):
+ """Iterate over probes grouped by chromosome name."""
+ uniq_chrom, idx = numpy.unique(self.chromosome, True)
+ sort_idx = idx.argsort()
+ idx = idx.take(sort_idx)
+ uniq_chrom = uniq_chrom.take(sort_idx)
+ idx2 = numpy.concatenate((idx[1:], [len(self.data)]))
+ for chrom, start, end in zip(uniq_chrom, idx, idx2):
+ subarr = self.to_array(self.data[start:end])
+ yield chrom, subarr
+
+ def by_gene(self, ignore=('-', 'CGH', '.')):
+ """Iterate over probes grouped by gene name.
+
+ Emits pairs of (gene name, CNA of rows with same name)
+
+ Groups each series of intergenic bins as a 'Background' gene; any
+ 'Background' bins within a gene are grouped with that gene.
+ Bins with names in `ignore` are treated as 'Background' bins, but retain
+ their name.
+ """
+ curr_chrom = None
+ curr_bg_rows_idx = []
+ curr_gene_name = None
+ curr_gene_rows_idx = []
+
+ for i, row in enumerate(self.data):
+ gene = str(row['gene'])
+ if gene == 'Background' or gene in ignore:
+ # This background *may* be in an intergenic region
+ if curr_chrom != row['chromosome']:
+ # New chromosome (not intergenic): emit the BG & reset
+ if curr_bg_rows_idx:
+ yield ('Background',
+ self.to_array(self.data.take(curr_bg_rows_idx)))
+ curr_bg_rows_idx = []
+ # Include this row in the current background
+ curr_chrom = row['chromosome']
+ curr_bg_rows_idx.append(i)
+ elif gene == curr_gene_name:
+ # Continue the current gene
+ # Any "background" was intronic; include in the current gene
+ if curr_bg_rows_idx:
+ curr_gene_rows_idx.extend(curr_bg_rows_idx)
+ curr_bg_rows_idx = []
+ # Add this row to the current gene
+ curr_gene_rows_idx.append(i)
+ else:
+ # New gene
+ # Emit the last gene, if any
+ if curr_gene_rows_idx:
+ yield (curr_gene_name,
+ self.to_array(self.data.take(curr_gene_rows_idx)))
+ # Emit the subsequent background, if any
+ if curr_bg_rows_idx:
+ yield ('Background',
+ self.to_array(self.data.take(curr_bg_rows_idx)))
+ # Reset BG trackers
+ curr_bg_rows_idx = []
+ # Start tracking the new gene
+ curr_gene_rows_idx = [i]
+ curr_chrom = row['chromosome']
+ curr_gene_name = gene
+
+ # Remainder
+ if curr_gene_rows_idx:
+ yield (curr_gene_name,
+ self.to_array(self.data.take(curr_gene_rows_idx)))
+ if curr_bg_rows_idx:
+ yield 'Background', self.to_array(self.data.take(curr_bg_rows_idx))
+
+ def by_segment(self, segments):
+ """Group cnarray rows by the segments that row midpoints land in.
+
+ Returns an iterable of segments and rows grouped by overlap with each
+ segment.
+
+ Note that segments don't necessarily cover all probes (some near
+ telo/centromeres may have been dropped as outliers during segmentation).
+ These probes are grouped with the nearest segment, so the endpoint of
+ the first/last probe may not match the corresponding segment endpoint.
+ This is appropriate if the segments were obtained from this probe array.
+ """
+ curr_probes_idx = []
+ segments = iter(segments)
+ curr_segment = next(segments)
+ next_segment = None
+ for i, row in enumerate(self.data):
+ probe_midpoint = 0.5 * (row['start'] + row['end'])
+ if (row['chromosome'] == curr_segment['chromosome'] and
+ curr_segment['start'] <= probe_midpoint <= curr_segment['end']):
+ # Probe is within the current segment
+ curr_probes_idx.append(i)
+
+ elif row['chromosome'] != curr_segment['chromosome']:
+ # Probe should be at the start of the next chromosome.
+ # Find the matching segment.
+ if next_segment is None:
+ next_segment = next(segments)
+
+ # Skip any extra segments on the chromosome after the current
+ # probes (e.g. probes are targets, trailing segments are based
+ # on antitargets alone)
+ while row['chromosome'] != next_segment['chromosome']:
+ try:
+ next_segment = next(segments)
+ except StopIteration:
+ raise ValueError("Segments are missing chromosome %r"
+ % row['chromosome'])
+
+ # Emit the current (completed) group
+ yield (curr_segment,
+ self.to_array(self.data.take(curr_probes_idx)))
+ # Begin a new group of probes
+ curr_segment, next_segment = next_segment, None
+ curr_probes_idx = [i]
+
+ elif row['start'] < curr_segment['start']:
+ # Probe is near the start of the current chromosome, but we've
+ # already seen another outlier here (rare/nonexistent case).
+ # Group with the current (upcoming) segment.
+ curr_probes_idx.append(i)
+
+ elif row['end'] > curr_segment['end']:
+ # Probe is at the end of an accessible region (e.g. p or q arm)
+ # on the current chromosome.
+ # Group this probe with whichever of the current or next
+ # segments is closer.
+ if next_segment is None:
+ next_segment = next(segments)
+ if (next_segment['chromosome'] != row['chromosome']
+ or (next_segment['start'] - probe_midpoint) >
+ (probe_midpoint - curr_segment['end'])):
+ # The current segment is closer than the next. Group here.
+ curr_probes_idx.append(i)
+ else:
+ # The next segment is closer. Emit the current group
+ # Begin a new group of probes
+ yield (curr_segment,
+ self.to_array(self.data.take(curr_probes_idx)))
+ # Reset/update trackers for the next group of probes
+ curr_segment, next_segment = next_segment, None
+ curr_probes_idx = [i]
+ else:
+ raise ValueError("Mismatch between probes and segments\n" +
+ "Probe: %s\nSegment: %s"
+ % (row2label(row), row2label(curr_segment)))
+ # Emit the remaining probes
+ yield curr_segment, self.to_array(self.data.take(curr_probes_idx))
+
+ def center_all(self, peak=False):
+ """Recenter coverage values to the autosomes' average (in-place)."""
+ chr_x = core.guess_chr_x(self)
+ chr_y = ('chrY' if chr_x.startswith('chr') else 'Y')
+ mask_autosome = ((self.chromosome != chr_x) &
+ (self.chromosome != chr_y))
+ mid = numpy.median(self.coverage[mask_autosome])
+ mask_cvg = (mask_autosome &
+ (self.coverage >= mid - 1.1) &
+ (self.coverage <= mid + 1.1))
+ if peak and sum(mask_cvg) > 210:
+ # Estimate the location of peak density
+ # hack: from a smoothed histogram -- enh: kernel density estimate
+ x = self.coverage[mask_cvg]
+ w = self['weight'][mask_cvg] if 'weight' in self else None
+ resn = int(round(numpy.sqrt(len(x))))
+ x_vals, x_edges = numpy.histogram(x, bins=8*resn, weights=w)
+ xs = smoothing.smoothed(x_vals, resn)
+ # DBG: Check the fit
+ # ngfrills.echo("Centering: median", mid,
+ # ", mode", x_edges[numpy.argmax(xs)],
+ # ", resolution", x_edges[2] - x_edges[1])
+ # from matplotlib import pyplot
+ # _fig, ax = pyplot.subplots()
+ # ax.plot(x_vals, c='k', alpha=.5)
+ # ax.plot(xs, c='r', lw=2)
+ # median_idx = x_edges.searchsorted(mid)
+ # ax.axvspan(median_idx - 1, median_idx, color='teal', alpha=.3)
+ # ax.vlines(numpy.argmax(xs), *ax.get_ylim(), colors='salmon')
+ # pyplot.show()
+ # --
+ mid = x_edges[numpy.argmax(xs)]
+ self.data['coverage'] -= mid
+
+ def copy(self):
+ """Create an independent copy of this object."""
+ return self.to_rows(numpy.copy(self.data))
+
+ def add_columns(self, **columns):
+ """Create a new CNA, adding the specified extra columns to this CNA."""
+ cols = {key: self.data[key]
+ for key in self.data.dtype.names}
+ cols.update(columns)
+ return self.__class__.from_columns(self.sample_id, **cols)
+
+ def drop_extra_columns(self):
+ """Remove any optional columns from this CopyNumArray.
+
+ Returns a new copy with only the core columns retained:
+ log2 value, chromosome, start, end, bin name.
+ """
+ result = self.__class__(self.sample_id)
+ result.data = rfn.drop_fields(self.data, self._xtra)
+ return result
+
+ def merge(self, other):
+ """Combine this array's data with another CopyNumArray (in-place).
+
+ Any optional columns must match between both arrays.
+ """
+ assert isinstance(other, CopyNumArray), (
+ "Argument (type %s) is not a CopyNumArray instance" % type(other))
+ self.data = numpy.concatenate((self.data, other.data))
+ self.sort()
+
+ def in_range(self, chrom, start=0, end=None, trim=False):
+ """Get the CopyNumArray portion within the given genomic range.
+
+ If trim=True, include bins straddling the range boundaries, and trim
+ the bins endpoints to the boundaries.
+ """
+ rows = self.data[self.chromosome == chrom]
+ if not len(rows):
+ raise ValueError("Chromosome %s is not in this probe set" % chrom)
+ if start:
+ if trim:
+ # Include all rows overlapping the start point
+ rows = rows[rows['end'].searchsorted(start, 'right'):]
+ # Update 5' endpoints to the boundary
+ rows['start'][rows['start'] < start] = start
+ else:
+ # Only rows entirely after the start point
+ rows = rows[rows['start'].searchsorted(start):]
+ if end:
+ if trim:
+ rows = rows[:rows['start'].searchsorted(end)]
+ # Update 3' endpoints to the boundary
+ rows['end'][rows['end'] > end] = end
+ else:
+ rows = rows[:rows['end'].searchsorted(end, 'right')]
+ return self.to_rows(rows)
+
+ def select(self, selector=None, **kwargs):
+ """Take a subset of rows where the given condition is true.
+
+ Arguments can be a function (lambda expression) returning a bool, which
+ will be used to select True rows, and/or keyword arguments like
+ gene="Background" or chromosome="chr7", which will select rows where the
+ keyed field equals the specified value.
+ """
+ outrows = self.data
+ if selector is not None:
+ assert callable(selector)
+ condition = numpy.apply_along_axis(selector, 0, outrows)
+ outrows = numpy.extract(condition, outrows)
+ for key, val in kwargs.items():
+ assert key in self
+ outrows = outrows[outrows[key] == val]
+ return self.to_rows(outrows)
+
+ def shuffle(self):
+ """Randomize the order of bins in this array (in-place)."""
+ numpy.random.seed(0xA5EED) # For reproducible results
+ order = numpy.arange(len(self.data))
+ numpy.random.shuffle(order)
+ self.data = self.data[order]
+ return order
+
+ def sort(self, key=None):
+ """Sort the bins in this array (in-place).
+
+ Optional argument 'key' is one of:
+
+ - a function that computes a sorting key from a CopyNumArray row
+ - a string identifier for an existing data column
+ - a list/array/iterable of precomputed keys equal in length to the
+ number of rows in this CopyNumArray.
+
+ By default, bins are sorted by chromosomal coordinates.
+ """
+ if key is None:
+ # Sort by chrom, then by start position
+ chrom_keys = list(map(core.sorter_chrom, self.data['chromosome']))
+ order = numpy.lexsort((self.data['start'], chrom_keys))
+ else:
+ # Sort by the given key, using a stable sort algorithm
+ if isinstance(key, basestring):
+ keys = self.data[key]
+ elif callable(key):
+ keys = list(map(key, self.data))
+ else:
+ if not len(key) == len(self):
+ raise ValueError("Sort key, as an array, must have the "
+ "same length as the CopyNumArray to sort "
+ "(%d vs. %d)." % (len(key), len(self)))
+ keys = key
+ order = numpy.argsort(keys, kind='mergesort')
+ self.data = self.data.take(order)
+
+ def squash_genes(self, ignore=('-', 'CGH', '.'), squash_background=False,
+ summary_stat=metrics.biweight_location):
+ """Combine consecutive bins with the same targeted gene name.
+
+ The `ignore` parameter lists bin names that not be counted as genes to
+ be output.
+
+ Parameter `summary_stat` is a function that summarizes an array of
+ coverage values to produce the "squashed" gene's coverage value. By
+ default this is the biweight location, but you might want median, mean,
+ max, min or something else in some cases.
+
+ Optional columns, if present, are dropped.
+ """
+ def squash_rows(name, rows):
+ """Combine multiple rows (for the same gene) into one row."""
+ chrom = core.check_unique(rows['chromosome'], 'chromosome')
+ start = rows[0]['start']
+ end = rows[-1]['end']
+ cvg = summary_stat(rows['coverage'])
+ outrow = [chrom, start, end, name, cvg]
+ # Handle extra fields
+ # ENH - no coverage stat; do weighted average as appropriate
+ for xfield in ('gc', 'rmask', 'spread', 'weight'):
+ if xfield in self:
+ outrow.append(summary_stat(rows[xfield]))
+ if 'probes' in self:
+ outrow.append(sum(rows['probes']))
+ return tuple(outrow)
+
+ outrows = []
+ for name, subarr in self.by_gene(ignore):
+ if name == 'Background' and not squash_background:
+ outrows.extend(map(tuple, subarr))
+ else:
+ outrows.append(squash_rows(name, subarr))
+ return self.to_rows(outrows)
+
+ def to_array(self, array):
+ return self.__class__.from_array(self.sample_id, array)
+
+ def to_rows(self, rows):
+ """Like from_rows, reusing this instance's metadata."""
+ return self.__class__.from_rows(self.sample_id, rows, self._xtra)
+
+ # I/O
+
+ @classmethod
+ def read(cls, infile, sample_id=None):
+ """Parse a tabular table of coverage data from a handle or filename."""
+ if sample_id is None:
+ if isinstance(infile, basestring):
+ sample_id = core.fbase(infile)
+ else:
+ sample_id = '<unknown>'
+ with as_handle(infile) as handle:
+ try:
+ header = next(handle)
+ except StopIteration:
+ # Don't crash on empty files
+ return cls(sample_id)
+ # Build CNA...
+ xtra = _sniff_xtra(header)
+ cnarr = cls(sample_id, xtra)
+ arr = numpy.loadtxt(handle, delimiter="\t", dtype=cnarr.data.dtype, ndmin=1)
+ cnarr.data = arr
+ return cnarr
+
+ def write(self, outfile=sys.stdout):
+ """Write coverage data to a file or handle in tabular format.
+
+ This is similar to BED or BedGraph format, but with extra columns.
+
+ To combine multiple samples in one file and/or convert to another
+ format, see the 'export' subcommand.
+ """
+ colnames = ['chromosome', 'start', 'end', 'gene', 'log2']
+ colnames.extend(self._xtra)
+ rows = (list(map(str, row)) for row in self.data)
+ with ngfrills.safe_write(outfile) as handle:
+ header = '\t'.join(colnames) + '\n'
+ handle.write(header)
+ handle.writelines('\t'.join(row) + '\n' for row in rows)
+
+
+def _sniff_xtra(header_line):
+ colnames = tuple(map(str.strip, header_line.split('\t')))
+ assert colnames[:5] == ('chromosome', 'start', 'end', 'gene', 'log2'), \
+ colnames[:5]
+ xtra = colnames[5:]
+ assert all(x in ('gc', 'rmask', 'spread', 'weight', 'probes')
+ for x in xtra)
+ return xtra
+
+
+def row2label(row):
+ return "{}:{}-{}".format(row['chromosome'], row['start'], row['end'])
diff --git a/cnvlib/_version.py b/cnvlib/_version.py
new file mode 100644
index 0000000..9ff9982
--- /dev/null
+++ b/cnvlib/_version.py
@@ -0,0 +1 @@
+__version__ = "0.7.11"
diff --git a/cnvlib/access.py b/cnvlib/access.py
new file mode 100755
index 0000000..db075b5
--- /dev/null
+++ b/cnvlib/access.py
@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+
+"""List the locations of accessible sequence regions in a FASTA file.
+
+Inaccessible regions, e.g. telomeres and centromeres, are masked out with N in
+the reference genome sequence; this script scans those to identify the
+coordinates of the accessible regions (those between the long spans of N's).
+"""
+
+import itertools
+import logging
+
+import numpy as np
+
+from cnvlib.ngfrills import parse_regions
+
+
+def log_this(chrom, run_start, run_end):
+ """Log a coordinate range, then return it as a tuple."""
+ logging.info("\tAccessible region %s:%d-%d (size %d)",
+ chrom, run_start, run_end, run_end - run_start)
+ return (chrom, run_start, run_end)
+
+
+def get_regions(fasta_fname):
+ """Find accessible sequence regions (those not masked out with 'N')."""
+ with open(fasta_fname) as infile:
+ chrom = cursor = run_start = None
+ for line in infile:
+ if line.startswith('>'):
+ # Emit the last chromosome's last run, if any
+ if run_start is not None:
+ yield log_this(chrom, run_start, cursor)
+ # Start new chromosome
+ chrom = line.split(None, 1)[0][1:]
+ run_start = None
+ cursor = 0
+ logging.info("%s: Scanning for accessible regions", chrom)
+ else:
+ line = line.rstrip()
+ if 'N' in line:
+ if all(c == 'N' for c in line):
+ # Shortcut if the line is all N chars
+ if run_start is not None:
+ yield log_this(chrom, run_start, cursor)
+ run_start = None
+ else:
+ # Slow route: line is a mix of N and non-N chars
+ line_chars = np.array(line, dtype='c')
+ n_indices = np.where(line_chars == 'N')[0]
+ # Emit the first block of non-N chars, if any
+ if run_start is not None:
+ yield log_this(chrom, run_start, cursor + n_indices[0])
+ elif n_indices[0] != 0:
+ yield log_this(chrom, cursor, cursor + n_indices[0])
+ # Emit any short intermediate blocks
+ gap_mask = np.diff(n_indices) > 1
+ if gap_mask.any():
+ ok_starts = n_indices[gap_mask] + 1 + cursor
+ ok_ends = n_indices[1:][gap_mask] + cursor
+ for start, end in zip(ok_starts, ok_ends):
+ yield log_this(chrom, start, end)
+ # Account for any tailing non-N chars
+ if n_indices[-1] + 1 < len(line_chars):
+ run_start = cursor + n_indices[-1] + 1
+ else:
+ run_start = None
+ else:
+ if run_start is None:
+ # Start of a new run of non-N characters
+ run_start = cursor
+ cursor += len(line)
+ # Emit the last run if it's accessible (i.e. not a telomere)
+ if run_start is not None:
+ yield log_this(chrom, run_start, cursor)
+
+
+def group_regions_by_chromosome(rows):
+ """Iterate through BED3 rows: (chrom, BED3-rows-in-this-chrom)"""
+ for chrom, rows in itertools.groupby(rows, lambda bed3: bed3[0]):
+ yield chrom, [(start, end) for _chrom, start, end in rows]
+
+
+def join_regions(regions, min_gap_size):
+ """Filter regions, joining those separated by small gaps."""
+ for chrom, coords in group_regions_by_chromosome(regions):
+ logging.info("%s: Joining over small gaps", chrom)
+ coords = iter(coords)
+ prev_start, prev_end = next(coords)
+ for start, end in coords:
+ gap = start - prev_end
+ assert gap > 0, ("Impossible gap between %s %d-%d and %d-%d (=%d)"
+ % (chrom, prev_start, prev_end, start, end, gap))
+ if gap < min_gap_size:
+ # Join with the previous region
+ logging.info("\tJoining %s %d-%d and %d-%d (gap size %d)",
+ chrom, prev_start, prev_end, start, end, gap)
+ prev_end = end
+ else:
+ # Keep the gap; emit the previous region as-is
+ logging.info("\tKeeping gap %s:%d-%d (size %d)",
+ chrom, prev_end, start, gap)
+ yield (chrom, prev_start, prev_end)
+ prev_start, prev_end = start, end
+ yield (chrom, prev_start, prev_end)
+
+
+def exclude_regions(bed_fname, access_rows):
+ ex_by_chrom = dict(group_regions_by_chromosome(
+ parse_regions(bed_fname, coord_only=True)))
+ if len(ex_by_chrom) == 0:
+ # Nothing to exclude -> emit the input regions unmodified
+ for row in access_rows:
+ yield row
+ else:
+ # Check if each input region overlaps an excluded region
+ for chrom, a_rows in group_regions_by_chromosome(access_rows):
+ if chrom in ex_by_chrom:
+ logging.info("%s: Subtracting excluded regions", chrom)
+ exclude_rows = iter(ex_by_chrom[chrom])
+ ex_start, ex_end = next_or_inf(exclude_rows)
+ for a_start, a_end in a_rows:
+ for row in exclude_in_region(exclude_rows, chrom, a_start,
+ a_end, ex_start, ex_end):
+ yield row
+ else:
+ logging.info("%s: No excluded regions", chrom)
+ for a_start, a_end in a_rows:
+ yield (chrom, a_start, a_end)
+
+
+def exclude_in_region(exclude_rows, chrom, a_start, a_end, ex_start, ex_end):
+ """Take region exclusions from an iterable and apply, perhaps recursively.
+
+ Returns an iterable (usually length 1) of two tuples:
+ (accessible chromosome, start, end)
+ (current exclusion start, end)
+ """
+ # If we've leapfrogged the excluded area, catch up
+ while ex_end <= a_start:
+ ex_start, ex_end = next_or_inf(exclude_rows)
+ if a_end <= ex_start:
+ # Excluded area does not overlap this one
+ yield (chrom, a_start, a_end)
+ else:
+ # Excluded area overlaps this one -> trim this region
+ logging.info("\tExclusion %s:%d-%d overlaps accessible region %d-%d",
+ chrom, ex_start, ex_end, a_start, a_end)
+ if ex_start <= a_start:
+ if ex_end < a_end:
+ # Exclusion covers this region's left (start) edge only
+ for row in exclude_in_region(exclude_rows, chrom, ex_end, a_end,
+ ex_start, ex_end):
+ yield row
+ # Otherwise: Exclusion covers the whole region
+ else:
+ yield (chrom, a_start, ex_start)
+ if ex_end < a_end:
+ # Exclusion is in the middle of this region
+ for row in exclude_in_region(exclude_rows, chrom, ex_end,
+ a_end, ex_start, ex_end):
+ yield row
+ # Otherwise: Exclusion covers this region's right (end) edge only
+
+
+def next_or_inf(iterable):
+ try:
+ return next(iterable)
+ except StopIteration:
+ return float("Inf"), float("Inf")
+
diff --git a/cnvlib/antitarget.py b/cnvlib/antitarget.py
new file mode 100644
index 0000000..1dc70b4
--- /dev/null
+++ b/cnvlib/antitarget.py
@@ -0,0 +1,136 @@
+"""Supporting functions for the 'antitarget' command."""
+from __future__ import absolute_import, division, print_function
+import re
+import sys
+import logging
+
+from Bio._py3k import range
+iteritems = (dict.iteritems if sys.version_info[0] < 3 else dict.items)
+
+from .params import INSERT_SIZE
+from .rary import RegionArray as RA
+
+
+def get_background(target_bed, access_bed, avg_bin_size, min_bin_size):
+ """Generate background intervals from target intervals.
+
+ Procedure:
+
+ - Invert target intervals
+ - Subtract the inverted targets from accessible regions
+ - For each of the resulting regions:
+
+ - Shrink by a fixed margin on each end
+ - If it's smaller than min_bin_size, skip
+ - Divide into equal-size (region_size/avg_bin_size) portions
+ - Emit the (chrom, start, end) coords of each portion
+ """
+ target_chroms = dict(RA.read(target_bed).by_chromosome())
+ if access_bed:
+ # Chromosomes' accessible sequence regions are given -- use them
+ access_chroms = dict(RA.read(access_bed).by_chromosome())
+ if access_chroms and set(access_chroms).isdisjoint(target_chroms):
+ raise ValueError("Chromosome names in the accessible regions file "
+ "%s %r do not match those in targets %s %r"
+ % (access_bed,
+ tuple(sorted(access_chroms.keys())[:3]),
+ target_bed,
+ tuple(sorted(target_chroms.keys())[:3])))
+ # But filter out untargeted alternative contigs and mitochondria
+ untgt_chroms = set(access_chroms) - set(target_chroms)
+ # Autosomes typically have numeric names, allosomes are X and Y
+ is_canonical = re.compile(r"(chr)?(\d+|[XYxy])$")
+ if any(is_canonical.match(c) for c in target_chroms):
+ chroms_to_skip = [c for c in untgt_chroms
+ if not is_canonical.match(c)]
+ else:
+ # Alternative contigs have long names -- skip them
+ max_tgt_chr_name_len = max(map(len, target_chroms))
+ chroms_to_skip = [c for c in untgt_chroms
+ if len(c) > max_tgt_chr_name_len]
+ for untgt_chr in chroms_to_skip:
+ logging.info("Skipping untargeted chromosome %s", untgt_chr)
+ del access_chroms[untgt_chr]
+ else:
+ # Chromosome accessible sequence regions not known -- use heuristics
+ # (chromosome length is endpoint of last probe; skip initial
+ # <magic number> of bases that are probably telomeric)
+ TELOMERE_SIZE = 150000
+ access_chroms = guess_chromosome_regions(target_chroms, TELOMERE_SIZE)
+
+ backgrounds = find_background_regions(access_chroms, target_chroms,
+ 2 * INSERT_SIZE)
+ bg_arr = RA.from_rows(backgrounds)
+ bg_arr.sort()
+
+ # Emit regions as antitarget bins according to avg_bin_size and min_bin_size
+ out_rows = []
+ for chrom, start, end in bg_arr.coords():
+ span = end - start
+ if span >= min_bin_size:
+ nbins = int(round(span / avg_bin_size)) or 1
+ if nbins == 1:
+ out_rows.append((chrom, start, end))
+ else:
+ # Divide the background region into equal-sized bins
+ bin_size = span / nbins
+ bin_start = start
+ bin_end = None
+ for i in range(1, nbins):
+ bin_end = start + int(i * bin_size)
+ out_rows.append((chrom, bin_start, bin_end))
+ bin_start = bin_end
+ out_rows.append((chrom, bin_start, end))
+ out_arr = RA.from_rows(out_rows)
+ out_arr["name"] = "Background"
+ return out_arr
+
+
+def guess_chromosome_regions(target_chroms, telomere_size):
+ """Determine (minimum) chromosome lengths from target coordinates."""
+ endpoints = [target_region[len(target_region) - 1, 'end']
+ for _chrom, target_region in target_chroms.iteritems()]
+ whole_chroms = RA.from_columns({"chromosome": target_chroms.keys(),
+ "start": telomere_size,
+ "end": endpoints})
+ return dict(whole_chroms.by_chromosome())
+
+
+def find_background_regions(access_chroms, target_chroms, pad_size):
+ """Take coordinates of accessible regions and targets; emit antitargets."""
+ for chrom, access_arr in access_chroms.iteritems():
+ if chrom in target_chroms:
+ target_regions = target_chroms[chrom].coords()
+
+ # Split each access_region at the targets it contains
+ _, tgt_start, tgt_end = next(target_regions)
+ assert tgt_start < tgt_end
+ for _, acc_start, acc_end in access_arr.coords():
+ if acc_end - acc_start <= 2 * pad_size:
+ # Accessible region is way too small
+ continue
+
+ bg_start = acc_start + pad_size
+ while tgt_start < acc_end:
+ # There may be at least one more target in this region
+ if tgt_end + pad_size > bg_start:
+ # Yes, there is a target in this region
+ if tgt_start - pad_size > bg_start:
+ # Split the background region at this target
+ yield (chrom, bg_start, tgt_start - pad_size)
+ bg_start = tgt_end + pad_size
+
+ # Done splitting that target; is there another?
+ try:
+ _, tgt_start, tgt_end = next(target_regions)
+ except StopIteration:
+ # Ensure all the remaining access_regions are unbroken
+ tgt_start = tgt_end = float('Inf')
+
+ # No remaining targets in this region - emit the rest of it
+ if acc_end - pad_size - bg_start > 0:
+ yield (chrom, bg_start, acc_end - pad_size)
+ else:
+ for _, acc_start, acc_end in access_arr.coords():
+ yield (chrom, acc_start + pad_size, acc_end - pad_size)
+
diff --git a/cnvlib/call.py b/cnvlib/call.py
new file mode 100644
index 0000000..4a0b3c1
--- /dev/null
+++ b/cnvlib/call.py
@@ -0,0 +1,223 @@
+"""Call copy number variants from segmented log2 ratios."""
+from __future__ import absolute_import, division, print_function
+
+import numpy as np
+import pandas as pd
+
+
+def log2_ratios(cnarr, absolutes, ploidy, is_reference_male,
+ min_abs_val=1e-3, round_to_int=False):
+ """Convert absolute copy numbers to log2 ratios.
+
+ Optionally round copy numbers to integers.
+
+ Account for reference gender & ploidy of sex chromosomes.
+ """
+ # Round absolute copy numbers to integer values
+ if round_to_int:
+ absolutes = np.rint(absolutes)
+ # Avoid a logarithm domain error
+ ratios = np.log2(np.maximum(absolutes / ploidy, min_abs_val))
+ # Adjust sex chromosomes to be relative to the reference
+ if is_reference_male:
+ ratios[np.asarray(cnarr.chromosome == cnarr._chr_x_label)] += 1.0
+ ratios[np.asarray(cnarr.chromosome == cnarr._chr_y_label)] += 1.0
+ return ratios
+
+
+def absolute_threshold(cnarr, ploidy, thresholds, is_reference_male):
+ """Call integer copy number using hard thresholds for each level.
+
+ Integer values are assigned for log2 ratio values less than each given
+ threshold value in sequence, counting up from zero.
+ Above the last threshold value, integer copy numbers are called assuming
+ full purity, diploidy, and rounding up.
+
+ Default thresholds follow this heuristic for calling CNAs in a tumor sample:
+ For single-copy gains and losses, assume 50% tumor cell clonality (including
+ normal cell contamination). Then::
+
+ R> log2(2:6 / 4)
+ -1.0 -0.4150375 0.0 0.3219281 0.5849625
+
+ Allowing for random noise of +/- 0.1, the cutoffs are::
+
+ DEL(0) < -1.1
+ LOSS(1) < -0.25
+ GAIN(3) >= +0.2
+ AMP(4) >= +0.7
+
+ For germline samples, better precision could be achieved with::
+
+ LOSS(1) < -0.4
+ GAIN(3) >= +0.3
+
+ """
+ absolutes = np.zeros(len(cnarr), dtype=np.float_)
+ for idx, row in enumerate(cnarr):
+ cnum = 0
+ ref_copies = _reference_copies_pure(row.chromosome, ploidy,
+ is_reference_male)
+ for cnum, thresh in enumerate(thresholds):
+ if row.log2 <= thresh:
+ if ref_copies != ploidy:
+ cnum = int(cnum * ref_copies / ploidy)
+ break
+ else:
+ cnum = int(np.ceil(_log2_ratio_to_absolute_pure(row.log2,
+ ref_copies)))
+ absolutes[idx] = cnum
+ return absolutes
+
+
+def absolute_clonal(cnarr, ploidy, purity, is_reference_male, is_sample_female):
+ """Calculate absolute copy number values from segment or bin log2 ratios."""
+ absolutes = np.zeros(len(cnarr), dtype=np.float_)
+ for i, row in enumerate(cnarr):
+ ref_copies, expect_copies = _reference_expect_copies(
+ row.chromosome, ploidy, is_sample_female, is_reference_male)
+ absolutes[i] = _log2_ratio_to_absolute(
+ row.log2, ref_copies, expect_copies, purity)
+ return absolutes
+
+
+def absolute_pure(cnarr, ploidy, is_reference_male):
+ """Calculate absolute copy number values from segment or bin log2 ratios."""
+ absolutes = np.zeros(len(cnarr), dtype=np.float_)
+ for i, row in enumerate(cnarr):
+ ref_copies = _reference_copies_pure(row.chromosome, ploidy,
+ is_reference_male)
+ absolutes[i] = _log2_ratio_to_absolute_pure(row.log2, ref_copies)
+ return absolutes
+
+
+def absolute_dataframe(cnarr, ploidy, purity, is_reference_male, is_sample_female):
+ """Absolute, expected and reference copy number in a DataFrame."""
+ absolutes = np.zeros(len(cnarr), dtype=np.float_)
+ reference_copies = expect_copies = np.zeros(len(cnarr), dtype=np.int_)
+ for i, row in enumerate(cnarr):
+ ref_copies, exp_copies = _reference_expect_copies(
+ row.chromosome, ploidy, is_sample_female, is_reference_male)
+ reference_copies[i] = ref_copies
+ expect_copies[i] = exp_copies
+ absolutes[i] = _log2_ratio_to_absolute(
+ row.log2, ref_copies, exp_copies, purity)
+ return pd.DataFrame({'absolute': absolutes,
+ 'reference': reference_copies,
+ 'expect': expect_copies})
+
+
+def absolute_expect(cnarr, ploidy, is_sample_female):
+ """Absolute integer number of expected copies in each bin.
+
+ I.e. the given ploidy for autosomes, and XY or XX sex chromosome counts
+ according to the sample's specified gender.
+ """
+ exp_copies = np.repeat(ploidy, len(cnarr))
+ is_y = np.asarray(cnarr.chromosome == cnarr._chr_y_label)
+ if is_sample_female:
+ exp_copies[is_y] = 0
+ else:
+ is_x = np.asarray(cnarr.chromosome == cnarr._chr_x_label)
+ exp_copies[is_x | is_y] = ploidy // 2
+ return exp_copies
+
+
+def absolute_reference(cnarr, ploidy, is_reference_male):
+ """Absolute integer number of reference copies in each bin.
+
+ I.e. the given ploidy for autosomes, 1 or 2 X according to the reference
+ gender, and always 1 copy of Y.
+ """
+ ref_copies = np.repeat(ploidy, len(cnarr))
+ is_x = np.asarray(cnarr.chromosome == cnarr._chr_x_label)
+ is_y = np.asarray(cnarr.chromosome == cnarr._chr_y_label)
+ if is_reference_male:
+ ref_copies[is_x] = ploidy // 2
+ ref_copies[is_y] = ploidy // 2
+ return ref_copies
+
+
+def _reference_expect_copies(chrom, ploidy, is_sample_female, is_reference_male):
+ """Determine the number copies of a chromosome expected and in reference.
+
+ For sex chromosomes, these values may not be the same ploidy as the
+ autosomes. The "reference" number is the chromosome's ploidy in the
+ CNVkit reference, while "expect" is the chromosome's neutral ploidy in the
+ given sample, based on the specified gender of each. E.g., given a female
+ sample and a male reference, on chromosome X the "reference" value is 1 but
+ "expect" is 2.
+
+ Return a pair of integers: number of copies in the reference, and expected in
+ the sample.
+ """
+ chrom = chrom.lower()
+ if chrom in ["chrx", "x"]:
+ ref_copies = (ploidy // 2 if is_reference_male else ploidy)
+ exp_copies = (ploidy if is_sample_female else ploidy // 2)
+ elif chrom in ["chry", "y"]:
+ ref_copies = ploidy // 2
+ exp_copies = (0 if is_sample_female else ploidy // 2)
+ else:
+ ref_copies = exp_copies = ploidy
+ return ref_copies, exp_copies
+
+
+def _reference_copies_pure(chrom, ploidy, is_reference_male):
+ """Determine the reference number of chromosome copies (pure sample).
+
+ Return the integer number of copies in the reference.
+ """
+ chrom = chrom.lower()
+ if chrom in ["chry", "y"] or (is_reference_male and chrom in ["chrx", "x"]):
+ ref_copies = ploidy // 2
+ else:
+ ref_copies = ploidy
+ return ref_copies
+
+
+def _log2_ratio_to_absolute(log2_ratio, ref_copies, expect_copies, purity=None):
+ """Transform a log2 ratio to absolute linear scale (for an impure sample).
+
+ Does not round to an integer absolute value here.
+
+ Math:
+
+ log2_ratio = log2(ncopies / ploidy)
+ 2^log2_ratio = ncopies / ploidy
+ ncopies = ploidy * 2^log2_ratio
+
+ With rescaling for purity:
+
+ let v = log2 ratio value, p = tumor purity,
+ r = reference ploidy, x = expected ploidy;
+ v = log_2(p*n/r + (1-p)*x/r)
+ 2^v = p*n/r + (1-p)*x/r
+ n*p/r = 2^v - (1-p)*x/r
+ n = (r*2^v - x*(1-p)) / p
+
+ If purity adjustment is skipped (p=1; e.g. if germline or if scaling for
+ heterogeneity was done beforehand):
+
+ n = r*2^v
+ """
+ if purity and purity < 1.0:
+ ncopies = (ref_copies * 2**log2_ratio - expect_copies * (1 - purity)
+ ) / purity
+ else:
+ ncopies = _log2_ratio_to_absolute_pure(log2_ratio, ref_copies)
+ return ncopies
+
+
+def _log2_ratio_to_absolute_pure(log2_ratio, ref_copies):
+ """Transform a log2 ratio to absolute linear scale (for a pure sample).
+
+ Purity adjustment is skipped. This is appropriate if the sample is germline
+ or if scaling for tumor heterogeneity was done beforehand.
+
+ Math::
+
+ n = r*2^v
+ """
+ ncopies = ref_copies * 2 ** log2_ratio
+ return ncopies
diff --git a/cnvlib/cnary.py b/cnvlib/cnary.py
new file mode 100644
index 0000000..6924756
--- /dev/null
+++ b/cnvlib/cnary.py
@@ -0,0 +1,298 @@
+"""CNVkit's core data structure, a copy number array."""
+from __future__ import print_function, absolute_import, division
+
+import logging
+
+import numpy as np
+import pandas as pd
+
+from . import core, gary, metrics, params, smoothing
+
+
+class CopyNumArray(gary.GenomicArray):
+ """An array of genomic intervals, treated like aCGH probes.
+
+ Required columns: chromosome, start, end, gene, log2
+
+ Optional columns: gc, rmask, spread, weight, probes
+ """
+ _required_columns = ("chromosome", "start", "end", "gene", "log2")
+ _required_dtypes = ("string", "int", "int", "string", "float")
+ # ENH: make gene optional
+ # Extra columns, in order:
+ # "gc", "rmask", "spread", "weight", "probes"
+
+ def __init__(self, data_table, meta_dict=None):
+ gary.GenomicArray.__init__(self, data_table, meta_dict)
+
+ @property
+ def log2(self):
+ return self.data["log2"]
+
+ @log2.setter
+ def log2(self, value):
+ self.data["log2"] = value
+
+ @property
+ def _chr_x_label(self):
+ if 'chr_x' in self.meta:
+ return self.meta['chr_x']
+ chr_x = ('chrX' if self[0, 'chromosome'].startswith('chr')
+ else 'X')
+ self.meta['chr_x'] = chr_x
+ return chr_x
+
+ @property
+ def _chr_y_label(self):
+ if 'chr_y' in self.meta:
+ return self.meta['chr_y']
+ chr_y = ('chrY' if self._chr_x_label.startswith('chr') else 'Y')
+ self.meta['chr_y'] = chr_y
+ return chr_y
+
+ # More meta to store:
+ # is_sample_male = bool
+ # is_reference_male = bool
+ # file_path
+ # * invalidate 'chr_x' if .chromosome/['chromosome'] is set
+
+ # Traversal
+
+ # XXX hair: some genes overlap; some bins cover multiple genes
+ # -> option: whether to split gene names on commas
+ def by_gene(self, ignore=params.IGNORE_GENE_NAMES):
+ """Iterate over probes grouped by gene name.
+
+ Emits pairs of (gene name, CNA of rows with same name)
+
+ Groups each series of intergenic bins as a 'Background' gene; any
+ 'Background' bins within a gene are grouped with that gene.
+ Bins with names in `ignore` are treated as 'Background' bins, but retain
+ their name.
+ """
+ start_idx = end_idx = None
+ for _chrom, subgary in self.by_chromosome():
+ prev_idx = 0
+ for gene in pd.unique(subgary.data['gene']):
+ if not (gene == 'Background' or gene in ignore):
+ gene_idx = (subgary.data['gene'] == gene).nonzero()[0]
+ if not len(gene_idx):
+ logging.warn("Specified gene name somehow missing: %s",
+ gene)
+ continue
+ start_idx = gene_idx[0]
+ end_idx = gene_idx[-1] + 1
+ if prev_idx < start_idx:
+ # Include intergenic regions
+ yield "Background", subgary.as_dataframe(
+ subgary.data.iloc[prev_idx:start_idx])
+ yield gene, subgary.as_dataframe(
+ subgary.data.iloc[start_idx:end_idx])
+ prev_idx = end_idx
+ if prev_idx < len(subgary) - 1:
+ # Include the telomere
+ yield "Background", subgary.as_dataframe(
+ subgary.data.iloc[prev_idx:])
+
+ # Manipulation
+
+ def center_all(self, estimator=np.median, skip_low=False):
+ """Recenter coverage values to the autosomes' average (in-place)."""
+ est_funcs = {
+ "mean": np.mean,
+ "median": np.median,
+ "mode": metrics.modal_location,
+ "biweight": metrics.biweight_location,
+ }
+ if isinstance(estimator, basestring):
+ if estimator in est_funcs:
+ estimator = est_funcs[estimator]
+ else:
+ raise ValueError("Estimator must be a function or one of: %s"
+ % ", ".join(map(repr, est_funcs)))
+ table = (self.drop_low_coverage() if skip_low else self).autosomes()
+ if table:
+ self.data['log2'] -= estimator(table['log2'])
+
+ def drop_low_coverage(self):
+ """Drop bins with extremely low log2 coverage values.
+
+ These are generally bins that had no reads mapped, and so were
+ substituted with a small dummy log2 value to avoid divide-by-zero
+ errors.
+ """
+ return self.as_dataframe(self.data[self.data['log2'] >
+ params.NULL_LOG2_COVERAGE -
+ params.MIN_REF_COVERAGE])
+
+ def squash_genes(self, summary_func=metrics.biweight_location,
+ squash_background=False, ignore=params.IGNORE_GENE_NAMES):
+ """Combine consecutive bins with the same targeted gene name.
+
+ The `ignore` parameter lists bin names that not be counted as genes to
+ be output.
+
+ Parameter `summary_func` is a function that summarizes an array of
+ coverage values to produce the "squashed" gene's coverage value. By
+ default this is the biweight location, but you might want median, mean,
+ max, min or something else in some cases.
+ """
+ def squash_rows(name, rows):
+ """Combine multiple rows (for the same gene) into one row."""
+ if len(rows) == 1:
+ return tuple(rows.iloc[0])
+ chrom = core.check_unique(rows.chromosome, 'chromosome')
+ start = rows.iloc[0]['start']
+ end = rows.iloc[-1]['end']
+ cvg = summary_func(rows.log2)
+ outrow = [chrom, start, end, name, cvg]
+ # Handle extra fields
+ # ENH - no coverage stat; do weighted average as appropriate
+ for xfield in ('gc', 'rmask', 'spread', 'weight'):
+ if xfield in self:
+ outrow.append(summary_func(rows[xfield]))
+ if 'probes' in self:
+ outrow.append(sum(rows['probes']))
+ return tuple(outrow)
+
+ outrows = []
+ for name, subarr in self.by_gene(ignore):
+ if name == 'Background' and not squash_background:
+ outrows.extend(subarr.data.itertuples(index=False))
+ else:
+ outrows.append(squash_rows(name, subarr.data))
+ return self.as_rows(outrows)
+
+ # Chromosomal gender
+
+ def shift_xx(self, male_reference=False):
+ """Adjust chrX coverages (divide in half) for apparent female samples."""
+ outprobes = self.copy()
+ is_xx = self.guess_xx(male_reference=male_reference)
+ if is_xx and male_reference:
+ # Female: divide X coverages by 2 (in log2: subtract 1)
+ outprobes[outprobes.chromosome == self._chr_x_label, 'log2'] -= 1.0
+ # Male: no change
+ elif not is_xx and not male_reference:
+ # Male: multiply X coverages by 2 (in log2: add 1)
+ outprobes[outprobes.chromosome == self._chr_x_label, 'log2'] += 1.0
+ # Female: no change
+ return outprobes
+
+ def guess_xx(self, male_reference=False, verbose=True):
+ """Guess whether a sample is female from chrX relative coverages.
+
+ Recommended cutoff values:
+ -0.5 -- raw target data, not yet corrected
+ +0.5 -- probe data already corrected on a male profile
+ """
+ cutoff = 0.5 if male_reference else -0.5
+ # ENH - better coverage approach: take Z-scores or rank of +1,0 or 0,-1
+ # based on the available probes, then choose which is more probable
+ rel_chrx_cvg = self.get_relative_chrx_cvg()
+ if rel_chrx_cvg is None:
+ return
+ is_xx = (rel_chrx_cvg >= cutoff)
+ if verbose:
+ logging.info("Relative log2 coverage of X chromosome: %g "
+ "(assuming %s)",
+ rel_chrx_cvg, ('male', 'female')[is_xx])
+ return is_xx
+
+ def get_relative_chrx_cvg(self):
+ """Get the relative log-coverage of chrX in a sample."""
+ chromosome_x = self[self.chromosome == self._chr_x_label]
+ if not len(chromosome_x):
+ logging.warn("*WARNING* No %s found in probes; check the input",
+ self._chr_x_label)
+ return
+ autosomes = self.autosomes()
+ auto_cvgs = autosomes['log2']
+ x_cvgs = chromosome_x['log2']
+ if 'probes' in self:
+ # Weight segments by number of probes to ensure good behavior
+ auto_sizes = autosomes['probes']
+ x_sizes = chromosome_x['probes']
+ # ENH: weighted median
+ rel_chrx_cvg = (np.average(x_cvgs, weights=x_sizes) -
+ np.average(auto_cvgs, weights=auto_sizes))
+ else:
+ rel_chrx_cvg = np.median(x_cvgs) - np.median(auto_cvgs)
+ return rel_chrx_cvg
+
+ def expect_flat_cvg(self, is_male_reference=None):
+ """Get the uninformed expected copy ratios of each bin.
+
+ Create an array of log2 coverages like a "flat" reference.
+
+ This is a neutral copy ratio at each autosome (log2 = 0.0) and sex
+ chromosomes based on whether the reference is male (XX or XY).
+ """
+ if is_male_reference is None:
+ is_male_reference = not self.guess_xx(verbose=False)
+ cvg = np.zeros(len(self), dtype=np.float_)
+ if is_male_reference:
+ # Single-copy X, Y
+ idx = np.asarray((self.chromosome == self._chr_x_label) |
+ (self.chromosome == self._chr_y_label))
+ else:
+ # Y will be all noise, so replace with 1 "flat" copy
+ idx = np.asarray(self.chromosome == self._chr_y_label)
+ cvg[idx] = -1.0
+ return cvg
+
+ # Reporting
+
+ def residuals(self, segments=None):
+ """Difference in log2 value of each bin from its segment mean.
+
+ If segments are just regions (e.g. RegionArray) with no log2 values
+ precalculated, subtract the median of this array's log2 values within
+ each region. If no segments are given, subtract each chromosome's
+ median.
+
+ """
+ if not segments:
+ resids = [subcna.log2 - subcna.log2.median()
+ for _chrom, subcna in self.by_chromosome()]
+ elif "log2" in segments:
+ resids = [subcna.log2 - seg.log2
+ for seg, subcna in self.by_ranges(segments)]
+ else:
+ resids = [subcna.log2 - subcna.log2.median()
+ for _seg, subcna in self.by_ranges(segments)]
+ return np.concatenate(resids) if resids else np.array([])
+
+ def guess_average_depth(self, segments=None, window=100):
+ """Estimate the effective average read depth from variance.
+
+ Assume read depths are Poisson distributed, converting log2 values to
+ absolute counts. Then the mean depth equals the variance , and the average
+ read depth is the estimated mean divided by the estimated variance.
+ Use robust estimators (Tukey's biweight location and midvariance) to
+ compensate for outliers and overdispersion.
+
+ With `segments`, take the residuals of this array's log2 values from
+ those of the segments to remove the confounding effect of real CNVs.
+
+ If `window` is an integer, calculate and subtract a smoothed trendline
+ to remove the effect of CNVs without segmentation (skipped if `segments`
+ are given).
+
+ See: http://www.evanmiller.org/how-to-read-an-unlabeled-sales-chart.html
+ """
+ # Try to drop allosomes
+ cnarr = self.autosomes()
+ if not len(cnarr):
+ cnarr = self
+ # Remove variations due to real/likely CNVs
+ y_log2 = cnarr.residuals(segments)
+ if window:
+ y_log2 -= smoothing.smoothed(y_log2, window)
+ # Guess Poisson parameter from absolute-scale values
+ y = np.exp2(y_log2)
+ # ENH: use weight argument to these stats
+ loc = metrics.biweight_location(y)
+ spread = metrics.biweight_midvariance(y, loc)
+ scale = loc / spread**2
+ return scale
diff --git a/cnvlib/commands.py b/cnvlib/commands.py
new file mode 100644
index 0000000..8571cdd
--- /dev/null
+++ b/cnvlib/commands.py
@@ -0,0 +1,1944 @@
+"""Command-line interface and corresponding API for CNVkit."""
+# NB: argparse CLI definitions and API functions are interwoven:
+# "_cmd_*" handles I/O and arguments processing for the command
+# "do_*" runs the command's functionality as an API
+from __future__ import absolute_import, division, print_function
+import argparse
+import collections
+import logging
+import os
+import sys
+
+import numpy as np
+import pandas as pd
+
+from Bio._py3k import map, range, zip
+iteritems = (dict.iteritems if sys.version_info[0] < 3 else dict.items)
+
+# If running headless, use a suitable GUI-less plotting backend
+if not os.environ.get('DISPLAY'):
+ import matplotlib
+ try:
+ matplotlib.use("Agg", force=True)
+ except TypeError:
+ # Older matplotlib doesn't have 'force' argument
+ matplotlib.use("Agg")
+
+from matplotlib import pyplot
+from matplotlib.backends.backend_pdf import PdfPages
+pyplot.ioff()
+
+from . import (core, ngfrills, parallel, params,
+ access, antitarget, call, coverage, export, fix, importers,
+ metrics, plots, reference, reports, segmentation, target)
+from .cnary import CopyNumArray as _CNA
+from .vary import VariantArray as _VA
+from .rary import RegionArray as _RA
+from ._version import __version__
+
+
+AP = argparse.ArgumentParser(
+ description="CNVkit, a command-line toolkit for copy number analysis.",
+ epilog="Contact Eric Talevich <eric.talevich at ucsf.edu> for help.")
+AP_subparsers = AP.add_subparsers(
+ help="Sub-commands (use with -h for more info)")
+
+
+# _____________________________________________________________________________
+# Core pipeline
+
+# batch -----------------------------------------------------------------------
+
+def _cmd_batch(args):
+ """Run the complete CNVkit pipeline on one or more BAM files."""
+ # Validate/restrict options, beyond what argparse mutual exclusion can do
+ if args.reference:
+ bad_flags = [flag
+ for is_used, flag in (
+ (args.normal is not None, '-n/--normal'),
+ (args.fasta, '-f/--fasta'),
+ (args.targets, '-t/--targets'),
+ (args.antitargets, '-a/--antitargets'),
+ (args.access, '-g/--access'),
+ (args.annotate, '--annotate'),
+ (args.short_names, '--short-names'),
+ (args.split, '--split'),
+ (args.target_avg_size, '--target-avg-size'),
+ (args.antitarget_avg_size, '--antitarget-avg-size'),
+ (args.antitarget_min_size, '--antitarget-min-size'),
+ ) if is_used]
+ if bad_flags:
+ sys.exit("If -r/--reference is given, options to construct a new " +
+ "reference (" + ", ".join(bad_flags) +
+ ") should not be used." +
+ "\n(See: cnvkit.py batch -h)")
+ elif not args.targets or args.normal is None:
+ sys.exit("Options -n/--normal and -t/--targets (at least) must be "
+ "given to build a new reference if -r/--reference is not used."
+ "\n(See: cnvkit.py batch -h)")
+
+ # Ensure sample IDs are unique to avoid overwriting outputs
+ seen_sids = {}
+ for fname in (args.bam_files or []) + (args.normal or []):
+ sid = core.fbase(fname)
+ if sid in seen_sids:
+ sys.exit("Duplicate sample ID %r (from %s and %s)"
+ % (sid, fname, seen_sids[sid]))
+ seen_sids[sid] = fname
+
+ if not args.reference:
+ # Build a copy number reference; update (anti)targets upon request
+ args.reference, args.targets, args.antitargets = batch_make_reference(
+ args.normal, args.targets, args.antitargets, args.male_reference,
+ args.fasta, args.annotate, args.short_names, args.split,
+ args.target_avg_size, args.access, args.antitarget_avg_size,
+ args.antitarget_min_size, args.output_reference, args.output_dir,
+ args.processes, args.count_reads)
+ elif args.targets is None and args.antitargets is None:
+ # Extract (anti)target BEDs from the given, existing CN reference
+ ref_arr = _CNA.read(args.reference)
+ target_coords, antitarget_coords = reference.reference2regions(ref_arr)
+ ref_pfx = os.path.join(args.output_dir, core.fbase(args.reference))
+ args.targets = ref_pfx + '.target-tmp.bed'
+ args.antitargets = ref_pfx + '.antitarget-tmp.bed'
+ core.write_tsv(args.targets, target_coords)
+ core.write_tsv(args.antitargets, antitarget_coords)
+
+ if args.bam_files:
+ logging.info("Running %d samples in %s",
+ len(args.bam_files),
+ ("serial" if args.processes == 1
+ else ("%d processes" % args.processes)))
+ pool = parallel.pick_pool(args.processes)
+ for bam in args.bam_files:
+ pool.apply_async(batch_run_sample,
+ (bam, args.targets, args.antitargets, args.reference,
+ args.output_dir, args.male_reference, args.scatter,
+ args.diagram, args.rlibpath, args.count_reads))
+ pool.close()
+ pool.join()
+
+
+def batch_make_reference(normal_bams, target_bed, antitarget_bed, male_reference,
+ fasta, annotate, short_names, split, target_avg_size,
+ access, antitarget_avg_size, antitarget_min_size,
+ output_reference, output_dir, processes, by_count):
+ """Build the CN reference from normal samples, targets and antitargets."""
+ # To make temporary filenames for processed targets or antitargets
+ tgt_name_base, tgt_name_ext = os.path.splitext(os.path.basename(target_bed))
+ if output_dir:
+ tgt_name_base = os.path.join(output_dir, tgt_name_base)
+
+ if annotate or short_names or split:
+ # Pre-process baits/targets
+ new_target_fname = tgt_name_base + '.target.bed'
+ tgt_rarr = do_targets(target_bed, annotate, short_names, split,
+ **({'avg_size': target_avg_size}
+ if split and target_avg_size
+ else {}))
+ tgt_rarr.write(new_target_fname, "bed4")
+ target_bed = new_target_fname
+
+ if not antitarget_bed:
+ # Build antitarget BED from the given targets
+ anti_kwargs = {}
+ if access:
+ anti_kwargs['access_bed'] = access
+ if antitarget_avg_size:
+ anti_kwargs['avg_bin_size'] = antitarget_avg_size
+ if antitarget_min_size:
+ anti_kwargs['min_bin_size'] = antitarget_min_size
+ anti_rarr = do_antitarget(target_bed, **anti_kwargs)
+ # Devise a temporary antitarget filename
+ antitarget_bed = tgt_name_base + '.antitarget.bed'
+ anti_rarr.write(antitarget_bed, "bed4")
+
+ if len(normal_bams) == 0:
+ logging.info("Building a flat reference...")
+ ref_arr = do_reference_flat(target_bed, antitarget_bed, fasta,
+ male_reference)
+ else:
+ logging.info("Building a copy number reference from normal samples...")
+ target_fnames = []
+ antitarget_fnames = []
+ # Run coverage on all normals
+ pool = parallel.pick_pool(processes)
+ for nbam in normal_bams:
+ sample_id = core.fbase(nbam)
+ sample_pfx = os.path.join(output_dir, sample_id)
+ tgt_fname = sample_pfx + '.targetcoverage.cnn'
+ pool.apply_async(batch_write_coverage,
+ (target_bed, nbam, tgt_fname, by_count))
+ target_fnames.append(tgt_fname)
+ anti_fname = sample_pfx + '.antitargetcoverage.cnn'
+ pool.apply_async(batch_write_coverage,
+ (antitarget_bed, nbam, anti_fname, by_count))
+ antitarget_fnames.append(anti_fname)
+ pool.close()
+ pool.join()
+ # Build reference from *.cnn
+ ref_arr = do_reference(target_fnames, antitarget_fnames, fasta,
+ male_reference)
+ if not output_reference:
+ output_reference = os.path.join(output_dir, "reference.cnn")
+ ngfrills.ensure_path(output_reference)
+ ref_arr.write(output_reference)
+
+ return output_reference, target_bed, antitarget_bed
+
+
+def batch_write_coverage(bed_fname, bam_fname, out_fname, by_count):
+ """Run coverage on one sample, write to file."""
+ cnarr = do_coverage(bed_fname, bam_fname, by_count)
+ cnarr.write(out_fname)
+
+
+def batch_run_sample(bam_fname, target_bed, antitarget_bed, ref_fname,
+ output_dir, male_reference=False, scatter=False,
+ diagram=False, rlibpath=None, by_count=False):
+ """Run the pipeline on one BAM file."""
+ # ENH - return probes, segments (cnarr, segarr)
+ logging.info("Running the CNVkit pipeline on %s ...", bam_fname)
+ sample_id = core.fbase(bam_fname)
+ sample_pfx = os.path.join(output_dir, sample_id)
+
+ raw_tgt = do_coverage(target_bed, bam_fname, by_count)
+ raw_tgt.write(sample_pfx + '.targetcoverage.cnn')
+
+ raw_anti = do_coverage(antitarget_bed, bam_fname, by_count)
+ raw_anti.write(sample_pfx + '.antitargetcoverage.cnn')
+
+ cnarr = do_fix(raw_tgt, raw_anti, _CNA.read(ref_fname))
+ cnarr.write(sample_pfx + '.cnr')
+
+ logging.info("Segmenting %s.cnr ...", sample_pfx)
+ segments = segmentation.do_segmentation(cnarr, 'cbs', rlibpath=rlibpath)
+ segments.write(sample_pfx + '.cns')
+
+ if scatter:
+ do_scatter(cnarr, segments)
+ pyplot.savefig(sample_pfx + '-scatter.pdf', format='pdf',
+ bbox_inches="tight")
+ logging.info("Wrote %s-scatter.pdf", sample_pfx)
+
+ if diagram:
+ from cnvlib import diagram
+ outfname = sample_pfx + '-diagram.pdf'
+ diagram.create_diagram(cnarr, segments, 0.5, 3, outfname,
+ male_reference)
+ logging.info("Wrote %s", outfname)
+
+
+P_batch = AP_subparsers.add_parser('batch', help=_cmd_batch.__doc__)
+P_batch.add_argument('bam_files', nargs='*',
+ help="Mapped sequence reads (.bam)")
+P_batch.add_argument('-y', '--male-reference', action='store_true',
+ help="""Use or assume a male reference (i.e. female samples will have +1
+ log-CNR of chrX; otherwise male samples would have -1 chrX).""")
+P_batch.add_argument('-c', '--count-reads', action='store_true',
+ help="""Get read depths by counting read midpoints within each bin.
+ (An alternative algorithm).""")
+P_batch.add_argument('-p', '--processes', type=int, default=1,
+ help="""Number of subprocesses used to running each of the BAM files in
+ parallel. Give 0 or a negative value to use the maximum number
+ of available CPUs. [Default: process each BAM in serial]""")
+P_batch.add_argument("--rlibpath",
+ help="Path to an alternative site-library to use for R packages.")
+
+# Reference-building options
+P_batch_newref = P_batch.add_argument_group(
+ "To construct a new copy number reference")
+P_batch_newref.add_argument('-n', '--normal', nargs='*',
+ help="""Normal samples (.bam) to construct the pooled reference.
+ If this option is used but no files are given, a "flat"
+ reference will be built.""")
+P_batch_newref.add_argument('-f', '--fasta',
+ help="Reference genome, FASTA format (e.g. UCSC hg19.fa)")
+P_batch_newref.add_argument('-t', '--targets', #required=True,
+ help="Target intervals (.bed or .list)")
+P_batch_newref.add_argument('-a', '--antitargets', #required=True,
+ help="Antitarget intervals (.bed or .list)")
+# For pre-processing targets
+P_batch_newref.add_argument('--annotate',
+ help="""UCSC refFlat.txt or ensFlat.txt file for the reference genome.
+ Pull gene names from this file and assign them to the target
+ regions.""")
+P_batch_newref.add_argument('--short-names', action='store_true',
+ help="Reduce multi-accession bait labels to be short and consistent.")
+P_batch_newref.add_argument('--split', action='store_true',
+ help="Split large tiled intervals into smaller, consecutive targets.")
+P_batch_newref.add_argument('--target-avg-size', type=int,
+ help="Average size of split target bins (results are approximate).")
+# For antitargets:
+P_batch_newref.add_argument('-g', '--access',
+ help="""Regions of accessible sequence on chromosomes (.bed), as
+ output by the 'access' command.""")
+P_batch_newref.add_argument('--antitarget-avg-size', type=int,
+ help="Average size of antitarget bins (results are approximate).")
+P_batch_newref.add_argument('--antitarget-min-size', type=int,
+ help="Minimum size of antitarget bins (smaller regions are dropped).")
+P_batch_newref.add_argument('--output-reference',
+ help="""Output filename/path for the new reference file being created.
+ (If given, ignores the -o/--output-dir option and will write the
+ file to the given path. Otherwise, \"reference.cnn\" will be
+ created in the current directory or specified output directory.)
+ """)
+
+P_batch_oldref = P_batch.add_argument_group("To reuse an existing reference")
+P_batch_oldref.add_argument('-r', '--reference', #required=True,
+ help="Copy number reference file (.cnn).")
+
+# Reporting options
+P_batch_report = P_batch.add_argument_group("Output options")
+P_batch_report.add_argument('-d', '--output-dir', default='.',
+ help="Output directory.")
+P_batch_report.add_argument('--scatter', action='store_true',
+ help="Create a whole-genome copy ratio profile as a PDF scatter plot.")
+P_batch_report.add_argument('--diagram', action='store_true',
+ help="Create a diagram of copy ratios on chromosomes as a PDF.")
+
+P_batch.set_defaults(func=_cmd_batch)
+
+
+# target ----------------------------------------------------------------------
+
+def _cmd_target(args):
+ """Transform bait intervals into targets more suitable for CNVkit."""
+ rarr = do_targets(args.interval, args.annotate, args.short_names,
+ args.split, args.avg_size)
+ rarr.write(args.output, "bed4")
+
+
+def do_targets(bed_fname, annotate=None, do_short_names=False, do_split=False,
+ avg_size=200/.75):
+ """Transform bait intervals into targets more suitable for CNVkit."""
+ bed_rows = ngfrills.parse_regions(bed_fname, False,
+ keep_strand=bool(annotate))
+ if annotate:
+ logging.info("Applying annotations as target names")
+ bed_rows = target.add_refflat_names(bed_rows, annotate)
+ if do_short_names:
+ logging.info("Shortening interval labels")
+ bed_rows = target.shorten_labels(bed_rows)
+ if do_split:
+ logging.info("Splitting large targets")
+ bed_rows = target.split_targets(bed_rows, avg_size)
+ bed_rarr = _RA.from_rows(bed_rows,
+ ['chromosome', 'start', 'end', 'name'])
+ bed_rarr.sort()
+ return bed_rarr
+
+
+P_target = AP_subparsers.add_parser('target', help=_cmd_target.__doc__)
+P_target.add_argument('interval',
+ help="""BED or interval file listing the targeted regions.""")
+P_target.add_argument('--annotate',
+ help="""UCSC refFlat.txt or ensFlat.txt file for the reference genome.
+ Pull gene names from this file and assign them to the target
+ regions.""")
+P_target.add_argument('--short-names', action='store_true',
+ help="Reduce multi-accession bait labels to be short and consistent.")
+P_target.add_argument('--split', action='store_true',
+ help="Split large tiled intervals into smaller, consecutive targets.")
+# Exons: [114--188==203==292--21750], mean=353 -> outlier=359, extreme=515
+# NV2: [65--181==190==239--12630], mean=264 -> outlier=277, extreme=364
+# Default avg_size chosen s.t. minimum bin size after split is ~= median
+P_target.add_argument('-a', '--avg-size', type=int, default=200 / .75,
+ help="""Average size of split target bins (results are approximate).
+ [Default: %(default)s]""")
+P_target.add_argument('-o', '--output', help="""Output file name.""")
+P_target.set_defaults(func=_cmd_target)
+
+
+# access ----------------------------------------------------------------------
+
+def _cmd_access(args):
+ """List the locations of accessible sequence regions in a FASTA file."""
+ # Closes over args.output
+ def write_row(chrom, run_start, run_end):
+ args.output.write("%s\t%s\t%s\n" % (chrom, run_start, run_end))
+ args.output.flush()
+
+ for row in do_access(args.fa_fname, args.exclude, args.min_gap_size):
+ write_row(*row)
+
+
+def do_access(fa_fname, exclude_fnames=(), min_gap_size=5000):
+ """List the locations of accessible sequence regions in a FASTA file."""
+ access_regions = access.get_regions(fa_fname)
+ for ex_fname in exclude_fnames:
+ access_regions = access.exclude_regions(ex_fname, access_regions)
+ for row in access.join_regions(access_regions, min_gap_size):
+ yield row
+
+
+P_access = AP_subparsers.add_parser('access', help=_cmd_access.__doc__)
+P_access.add_argument("fa_fname",
+ help="Genome FASTA file name")
+P_access.add_argument("-s", "--min-gap-size", type=int, default=5000,
+ help="""Minimum gap size between accessible sequence
+ regions. Regions separated by less than this distance will
+ be joined together. [Default: %(default)s]""")
+P_access.add_argument("-x", "--exclude", action="append", default=[],
+ help="""Additional regions to exclude, in BED format. Can be
+ used multiple times.""")
+P_access.add_argument("-o", "--output",
+ type=argparse.FileType('w'), default=sys.stdout,
+ help="Output file name")
+P_access.set_defaults(func=_cmd_access)
+
+
+# antitarget ------------------------------------------------------------------
+
+def _cmd_antitarget(args):
+ """Derive a background/antitarget BED file from a target BED file."""
+ out_rarr = do_antitarget(args.interval, args.access,
+ args.avg_size, args.min_size)
+
+ if not args.output:
+ base, ext = args.interval.rsplit('.', 1)
+ args.output = base + '.antitarget.' + ext
+ out_rarr.write(args.output, "bed4")
+
+
+def do_antitarget(target_bed, access_bed=None, avg_bin_size=100000,
+ min_bin_size=None):
+ """Derive a background/antitarget BED file from a target BED file."""
+ if not min_bin_size:
+ min_bin_size = 2 * int(avg_bin_size * (2 ** params.MIN_REF_COVERAGE))
+ return antitarget.get_background(target_bed, access_bed, avg_bin_size,
+ min_bin_size)
+
+
+P_anti = AP_subparsers.add_parser('antitarget', help=_cmd_antitarget.__doc__)
+P_anti.add_argument('interval',
+ help="""BED or interval file listing the targeted regions.""")
+P_anti.add_argument('-g', '--access',
+ help="""Regions of accessible sequence on chromosomes (.bed), as
+ output by genome2access.py.""")
+P_anti.add_argument('-a', '--avg-size', type=int, default=100000,
+ help="""Average size of antitarget bins (results are approximate).
+ [Default: %(default)s]""")
+P_anti.add_argument('-m', '--min-size', type=int,
+ help="""Minimum size of antitarget bins (smaller regions are dropped).
+ [Default: 1/16 avg size, calculated]""")
+P_anti.add_argument('-o', '--output', help="""Output file name.""")
+P_anti.set_defaults(func=_cmd_antitarget)
+
+
+# coverage --------------------------------------------------------------------
+
+def _cmd_coverage(args):
+ """Calculate coverage in the given regions from BAM read depths."""
+ pset = do_coverage(args.interval, args.bam_file, args.count, args.min_mapq)
+ if not args.output:
+ # Create an informative but unique name for the coverage output file
+ bambase = core.fbase(args.bam_file)
+ bedbase = core.rbase(args.interval)
+ tgtbase = ('antitargetcoverage'
+ if 'anti' in bedbase.lower()
+ else 'targetcoverage')
+ args.output = '%s.%s.cnn' % (bambase, tgtbase)
+ if os.path.exists(args.output):
+ args.output = '%s.%s.cnn' % (bambase, bedbase)
+ ngfrills.ensure_path(args.output)
+ pset.write(args.output)
+
+
+def do_coverage(bed_fname, bam_fname, by_count=False, min_mapq=0):
+ """Calculate coverage in the given regions from BAM read depths."""
+ if not ngfrills.ensure_bam_sorted(bam_fname):
+ raise RuntimeError("BAM file %s must be sorted by coordinates"
+ % bam_fname)
+ ngfrills.ensure_bam_index(bam_fname)
+ # ENH: count importers.TOO_MANY_NO_COVERAGE & warn
+ cnarr = coverage.interval_coverages(bed_fname, bam_fname, by_count,
+ min_mapq)
+ return cnarr
+
+
+P_coverage = AP_subparsers.add_parser('coverage', help=_cmd_coverage.__doc__)
+P_coverage.add_argument('bam_file', help="Mapped sequence reads (.bam)")
+P_coverage.add_argument('interval', help="Intervals (.bed or .list)")
+P_coverage.add_argument('-c', '--count', action='store_true',
+ help="""Get read depths by counting read midpoints within each bin.
+ (An alternative algorithm).""")
+P_coverage.add_argument('-q', '--min-mapq', type=int, default=0,
+ help="""Minimum mapping quality score (phred scale 0-60) to count a read
+ for coverage depth. [Default: %(default)s]""")
+P_coverage.add_argument('-o', '--output', help="""Output file name.""")
+P_coverage.set_defaults(func=_cmd_coverage)
+
+
+# reference -------------------------------------------------------------------
+
+def _cmd_reference(args):
+ """Compile a coverage reference from the given files (normal samples)."""
+ usage_err_msg = ("Give .cnn samples OR targets and antitargets.")
+ if args.targets and args.antitargets:
+ # Flat refence
+ assert not args.references, usage_err_msg
+ ref_probes = do_reference_flat(args.targets, args.antitargets,
+ args.fasta, args.male_reference)
+ elif args.references:
+ # Pooled reference
+ assert not args.targets and not args.antitargets, usage_err_msg
+ filenames = []
+ for path in args.references:
+ if os.path.isdir(path):
+ filenames.extend(os.path.join(path, f) for f in os.listdir(path)
+ if f.endswith('targetcoverage.cnn'))
+ else:
+ filenames.append(path)
+ targets = [f for f in filenames if 'antitarget' not in f]
+ antitargets = [f for f in filenames if 'antitarget' in f]
+ logging.info("Number of target and antitarget files: %d, %d",
+ len(targets), len(antitargets))
+ ref_probes = do_reference(targets, antitargets, args.fasta,
+ args.male_reference,
+ args.do_gc, args.do_edge, args.do_rmask)
+ else:
+ raise ValueError(usage_err_msg)
+
+ ref_fname = args.output or "cnv_reference.cnn"
+ ngfrills.ensure_path(ref_fname)
+ ref_probes.write(ref_fname)
+
+
+def do_reference(target_fnames, antitarget_fnames, fa_fname=None,
+ male_reference=False, do_gc=True, do_edge=True, do_rmask=True):
+ """Compile a coverage reference from the given files (normal samples)."""
+ core.assert_equal("Unequal number of target and antitarget files given",
+ targets=len(target_fnames),
+ antitargets=len(antitarget_fnames))
+ if not fa_fname:
+ logging.info("No FASTA reference genome provided; "
+ "skipping GC, RM calculations")
+
+ # Calculate & save probe centers
+ ref_probes = reference.combine_probes(target_fnames, fa_fname,
+ male_reference, True,
+ do_gc, do_edge, False)
+ ref_probes.add(reference.combine_probes(antitarget_fnames, fa_fname,
+ male_reference, False,
+ do_gc, False, do_rmask))
+ ref_probes.center_all(skip_low=True)
+ ref_probes.sort_columns()
+ reference.warn_bad_probes(ref_probes)
+ return ref_probes
+
+
+def do_reference_flat(targets, antitargets, fa_fname=None,
+ male_reference=False):
+ """Compile a neutral-coverage reference from the given intervals.
+
+ Combines the intervals, shifts chrX values if requested, and calculates GC
+ and RepeatMasker content from the genome FASTA sequence.
+ """
+ ref_probes = reference.bed2probes(targets)
+ ref_probes.add(reference.bed2probes(antitargets))
+ # Set sex chromosomes by "reference" gender
+ ref_probes['log2'] = ref_probes.expect_flat_cvg(male_reference)
+ # Calculate GC and RepeatMasker content for each probe's genomic region
+ if fa_fname:
+ gc, rmask = reference.get_fasta_stats(ref_probes, fa_fname)
+ ref_probes['gc'] = gc
+ ref_probes['rmask'] = rmask
+ reference.warn_bad_probes(ref_probes)
+ else:
+ logging.info("No FASTA reference genome provided; "
+ "skipping GC, RM calculations")
+ ref_probes.sort_columns()
+ return ref_probes
+
+
+P_reference = AP_subparsers.add_parser('reference', help=_cmd_reference.__doc__)
+P_reference.add_argument('references', nargs='*',
+ help="""Normal-sample target or antitarget .cnn files, or the
+ directory that contains them.""")
+P_reference.add_argument('-f', '--fasta',
+ help="Reference genome, FASTA format (e.g. UCSC hg19.fa)")
+P_reference.add_argument('-t', '--targets',
+ help="Target intervals (.bed or .list)")
+P_reference.add_argument('-a', '--antitargets',
+ help="Antitarget intervals (.bed or .list)")
+P_reference.add_argument('-y', '--male-reference', action='store_true',
+ help="""Create a male reference: shift female samples' chrX
+ log-coverage by -1, so the reference chrX average is -1.
+ Otherwise, shift male samples' chrX by +1, so the reference chrX
+ average is 0.""")
+P_reference.add_argument('--no-gc', dest='do_gc', action='store_false',
+ help="Skip GC correction.")
+P_reference.add_argument('--no-edge', dest='do_edge', action='store_false',
+ help="Skip edge-effect correction.")
+P_reference.add_argument('--no-rmask', dest='do_rmask', action='store_false',
+ help="Skip RepeatMasker correction.")
+P_reference.add_argument('-o', '--output', help="Output file name.")
+P_reference.set_defaults(func=_cmd_reference)
+
+
+# fix -------------------------------------------------------------------------
+
+def _cmd_fix(args):
+ """Combine target and antitarget coverages and correct for biases.
+
+ Adjust raw coverage data according to the given reference, correct potential
+ biases and re-center.
+ """
+ # Verify that target and antitarget are from the same sample
+ tgt_raw = _CNA.read(args.target)
+ anti_raw = _CNA.read(args.antitarget)
+ if tgt_raw.sample_id != anti_raw.sample_id:
+ raise ValueError("Sample IDs do not match:"
+ "'%s' (target) vs. '%s' (antitarget)"
+ % (tgt_raw.sample_id, anti_raw.sample_id))
+ target_table = do_fix(tgt_raw, anti_raw, _CNA.read(args.reference),
+ args.do_gc, args.do_edge, args.do_rmask)
+ target_table.write(args.output or tgt_raw.sample_id + '.cnr')
+
+
+def do_fix(target_raw, antitarget_raw, reference,
+ do_gc=True, do_edge=True, do_rmask=True):
+ """Combine target and antitarget coverages and correct for biases."""
+ # Load, recenter and GC-correct target & antitarget probes separately
+ logging.info("Processing target: %s", target_raw.sample_id)
+ cnarr = fix.load_adjust_coverages(target_raw, reference, True,
+ do_gc, do_edge, False)
+ logging.info("Processing antitarget: %s", antitarget_raw.sample_id)
+ anti_cnarr = fix.load_adjust_coverages(antitarget_raw, reference, False,
+ do_gc, False, do_rmask)
+ if len(anti_cnarr):
+ # Down-weight the more variable probe set (targets or antitargets)
+ tgt_iqr = metrics.interquartile_range(cnarr.drop_low_coverage().residuals())
+ anti_iqr = metrics.interquartile_range(anti_cnarr.drop_low_coverage().residuals())
+ iqr_ratio = max(tgt_iqr, .01) / max(anti_iqr, .01)
+ if iqr_ratio > 1:
+ logging.info("Targets are %.2f x more variable than antitargets",
+ iqr_ratio)
+ cnarr["weight"] /= iqr_ratio
+ else:
+ logging.info("Antitargets are %.2f x more variable than targets",
+ 1. / iqr_ratio)
+ anti_cnarr["weight"] *= iqr_ratio
+ # Combine target and antitarget bins
+ cnarr.add(anti_cnarr)
+ cnarr.center_all(skip_low=True)
+ return cnarr
+
+
+P_fix = AP_subparsers.add_parser('fix', help=_cmd_fix.__doc__)
+P_fix.add_argument('target',
+ help="Target coverage file (.targetcoverage.cnn).")
+P_fix.add_argument('antitarget',
+ help="Antitarget coverage file (.antitargetcoverage.cnn).")
+P_fix.add_argument('reference',
+ help="Reference coverage (.cnn).")
+# P_fix.add_argument('--do-gc', action='store_true', default=True,
+# help="Do GC correction.")
+# P_fix.add_argument('--do-edge', action='store_true',
+# help="Do edge-effect correction.")
+# P_fix.add_argument('--do-size', action='store_true',
+# help="Do interval-size correction.")
+P_fix.add_argument('--no-gc', dest='do_gc', action='store_false',
+ help="Skip GC correction.")
+P_fix.add_argument('--no-edge', dest='do_edge', action='store_false',
+ help="Skip edge-effect correction.")
+P_fix.add_argument('--no-rmask', dest='do_rmask', action='store_false',
+ help="Skip RepeatMasker correction.")
+P_fix.add_argument('-o', '--output',
+ help="Output file name.")
+P_fix.set_defaults(func=_cmd_fix)
+
+
+# segment ---------------------------------------------------------------------
+
+def _cmd_segment(args):
+ """Infer copy number segments from the given coverage table."""
+ cnarr = _CNA.read(args.filename)
+ variants = (_VA.read_vcf(args.vcf, skip_hom=True, skip_somatic=True)
+ if args.vcf else None)
+ results = segmentation.do_segmentation(cnarr, args.method, args.threshold,
+ variants=variants,
+ skip_low=args.drop_low_coverage,
+ skip_outliers=args.drop_outliers,
+ save_dataframe=bool(args.dataframe),
+ rlibpath=args.rlibpath)
+ if args.dataframe:
+ segments, dframe = results
+ with open(args.dataframe, 'w') as handle:
+ handle.write(dframe)
+ logging.info("Wrote %s", args.dataframe)
+ else:
+ segments = results
+ segments.write(args.output or segments.sample_id + '.cns')
+
+
+P_segment = AP_subparsers.add_parser('segment', help=_cmd_segment.__doc__)
+P_segment.add_argument('filename',
+ help="Bin-level log2 ratios (.cnr file), as produced by 'fix'.")
+P_segment.add_argument('-o', '--output',
+ help="Output table file name (CNR-like table of segments, .cns).")
+P_segment.add_argument('-d', '--dataframe',
+ help="""File name to save the raw R dataframe emitted by CBS or
+ Fused Lasso. (Useful for debugging.)""")
+P_segment.add_argument('-m', '--method',
+ choices=('cbs', 'haar', 'flasso'), default='cbs',
+ help="""Segmentation method (CBS, HaarSeg, or Fused Lasso).
+ [Default: %(default)s]""")
+P_segment.add_argument('-t', '--threshold', type=float,
+ help="""Significance threshold (p-value or FDR, depending on method) to
+ accept breakpoints during segmentation.""")
+P_segment.add_argument('-v', '--vcf',
+ help="""VCF file name containing variants for segmentation by allele
+ frequencies.""")
+P_segment.add_argument("--drop-low-coverage", action='store_true',
+ help="""Drop very-low-coverage bins before segmentation to avoid
+ false-positive deletions in poor-quality tumor samples.""")
+P_segment.add_argument("--drop-outliers",
+ type=float, default=10, metavar="FACTOR",
+ help="""Drop outlier bins more than this many multiples of the 95th
+ quantile away from the average within a rolling window.
+ Set to 0 for no outlier filtering.
+ [Default: %(default)g]""")
+P_segment.add_argument("--rlibpath",
+ help="Path to an alternative site-library to use for R packages.")
+P_segment.set_defaults(func=_cmd_segment)
+
+
+# rescale ---------------------------------------------------------------------
+
+def _cmd_rescale(args):
+ """[DEPRECATED] Rescale segment copy ratios given known purity and ploidy.
+
+ Instead, use the command "call -m none".
+ """
+ if args.purity and not 0.0 < args.purity <= 1.0:
+ raise RuntimeError("Purity must be between 0 and 1.")
+
+ cnarr = _CNA.read(args.filename)
+ if args.center:
+ cnarr.center_all(args.center)
+ if args.purity and args.purity < 1.0:
+ is_sample_female = verify_gender_arg(cnarr, args.gender,
+ args.male_reference)
+ cnarr = do_rescale(cnarr, args.ploidy, args.purity,
+ args.male_reference, is_sample_female)
+ cnarr.write(args.output)
+
+
+def do_rescale(cnarr, ploidy=2, purity=None, is_reference_male=False,
+ is_sample_female=False):
+ absolutes = call.absolute_clonal(cnarr, ploidy, purity,
+ is_reference_male, is_sample_female)
+ # Convert back to log2 ratios; avoid a logarithm domain error
+ outarr = cnarr.copy()
+ outarr['log2'] = call.log2_ratios(cnarr, absolutes, ploidy,
+ is_reference_male)
+ return outarr
+
+
+P_rescale = AP_subparsers.add_parser('rescale', help=_cmd_rescale.__doc__)
+P_rescale.add_argument('filename',
+ help="Copy ratios (.cnr or .cns).")
+P_rescale.add_argument("--center",
+ choices=('mean', 'median', 'mode', 'biweight'),
+ help="""Re-center the log2 ratio values using this estimate of the
+ center or average value.""")
+P_rescale.add_argument("--ploidy", type=int, default=2,
+ help="Ploidy of the sample cells. [Default: %(default)d]")
+P_rescale.add_argument("--purity", type=float,
+ help="Estimated tumor cell fraction, a.k.a. purity or cellularity.")
+P_rescale.add_argument("-g", "--gender",
+ choices=('m', 'male', 'Male', 'f', 'female', 'Female'),
+ help="""Specify the sample's gender as male or female. (Otherwise
+ guessed from chrX copy number).""")
+P_rescale.add_argument('-y', '--male-reference', action='store_true',
+ help="""Was a male reference used? If so, expect half ploidy on
+ chrX and chrY; otherwise, only chrY has half ploidy. In CNVkit,
+ if a male reference was used, the "neutral" copy number (ploidy)
+ of chrX is 1; chrY is haploid for either gender reference.""")
+P_rescale.add_argument('-o', '--output',
+ help="Output table file name (CNR-like table of segments, .cns).")
+P_rescale.set_defaults(func=_cmd_rescale)
+
+
+# call ------------------------------------------------------------------------
+
+def _cmd_call(args):
+ """Call copy number variants from segmented log2 ratios."""
+ if args.purity and not 0.0 < args.purity <= 1.0:
+ raise RuntimeError("Purity must be between 0 and 1.")
+
+ cnarr = _CNA.read(args.filename)
+ if args.center:
+ cnarr.center_all(args.center)
+ is_sample_female = (verify_gender_arg(cnarr, args.gender,
+ args.male_reference)
+ if args.purity and args.purity < 1.0
+ else None)
+ vcf = (_VA.read_vcf(args.vcf, skip_hom=True, skip_somatic=True)
+ if args.vcf
+ else None)
+ cnarr = do_call(cnarr, vcf, args.method, args.ploidy, args.purity,
+ args.male_reference, is_sample_female, args.thresholds)
+ cnarr.write(args.output or cnarr.sample_id + '.call.cns')
+
+
+def do_call(cnarr, variants=None, method="threshold", ploidy=2, purity=None,
+ is_reference_male=False, is_sample_female=False,
+ thresholds=(-1.1, -0.25, 0.2, 0.7)):
+ if method not in ("threshold", "clonal", "none"):
+ raise ValueError("Argument `method` must be one of: clonal, threshold")
+
+ outarr = cnarr.copy()
+ if variants:
+ # baf_median = lambda x: np.median(np.abs(x - .5)) + .5
+ baf_median = export.mirrored_baf_median
+ outarr["baf"] = outarr.match_to_bins(variants, 'alt_freq', np.nan,
+ summary_func=baf_median)
+
+ if purity and purity < 1.0:
+ logging.info("Rescaling sample with purity %g, ploidy %d",
+ purity, ploidy)
+ absolutes = call.absolute_clonal(outarr, ploidy, purity,
+ is_reference_male, is_sample_female)
+ # Recalculate sample log2 ratios after rescaling for purity
+ outarr["log2"] = call.log2_ratios(outarr, absolutes, ploidy,
+ is_reference_male)
+ if variants:
+ # Rescale b-allele frequencies for purity
+ outarr["baf"] = rescale_baf(purity, outarr["baf"])
+ elif method == "clonal":
+ # Estimate absolute copy numbers from the original log2 values
+ logging.info("Calling copy number with clonal ploidy %d", ploidy)
+ absolutes = call.absolute_pure(outarr, ploidy, is_reference_male)
+
+ if method == "threshold":
+ # Apply cutoffs to either original or rescaled log2 values
+ tokens = ["%g => %d" % (thr, i) for i, thr in enumerate(thresholds)]
+ logging.info("Calling copy number with thresholds: %s",
+ ", ".join(tokens))
+ absolutes = call.absolute_threshold(outarr, ploidy, thresholds,
+ is_reference_male)
+
+ if method != "none":
+ outarr["cn"] = np.asarray(np.rint(absolutes), dtype=np.int_)
+ if "baf" in outarr:
+ # Major and minor allelic copy numbers
+ outarr["cn1"] = np.asarray(np.rint(absolutes * outarr["baf"]),
+ dtype=np.int_).clip(0, outarr["cn"])
+ outarr["cn2"] = outarr["cn"] - outarr["cn1"]
+ is_null = outarr["baf"].isnull()
+ outarr[is_null, "cn1"] = np.nan
+ outarr[is_null, "cn2"] = np.nan
+ return outarr
+
+
+def rescale_baf(purity, observed_baf, normal_baf=0.5):
+ """Adjust B-allele frequencies for sample purity.
+
+ Math:
+ t_baf*purity + n_baf*(1-purity) = obs_baf
+ obs_baf - n_baf * (1-purity) = t_baf * purity
+ t_baf = (obs_baf - n_baf * (1-purity))/purity
+ """
+ # ENH: use normal_baf array if available
+ tumor_baf = (observed_baf - normal_baf * (1-purity)) / purity
+ # ENH: warn if tumor_baf < 0 -- purity estimate may be too low
+ return tumor_baf
+
+
+def csvstring(text):
+ return tuple(map(float, text.split(",")))
+
+
+def verify_gender_arg(cnarr, gender_arg, is_male_reference):
+ is_sample_female = cnarr.guess_xx(is_male_reference, verbose=False)
+ if gender_arg:
+ is_sample_female_given = (gender_arg.lower() not in ["m", "male"])
+ if is_sample_female != is_sample_female_given:
+ logging.info("Sample gender specified as %s "
+ "but chrX copy number looks like %s",
+ gender_arg,
+ "female" if is_sample_female else "male")
+ is_sample_female = is_sample_female_given
+ logging.info("Treating sample gender as %s",
+ "female" if is_sample_female else "male")
+ return is_sample_female
+
+
+P_call = AP_subparsers.add_parser('call', help=_cmd_call.__doc__)
+P_call.add_argument('filename',
+ help="Copy ratios (.cnr or .cns).")
+P_call.add_argument("--center",
+ choices=('mean', 'median', 'mode', 'biweight'),
+ help="""Re-center the log2 ratio values using this estimate of the
+ center or average value.""")
+P_call.add_argument('-m', '--method',
+ choices=('threshold', 'clonal', 'none'), default='threshold',
+ help="""Calling method. [Default: %(default)s]""")
+P_call.add_argument('-t', '--thresholds',
+ type=csvstring, default="-1.1,-0.25,0.2,0.7",
+ help="""Hard thresholds for calling each integer copy number, separated
+ by commas. Use the '=' sign on the command line, e.g.: -t=-1,0,1
+ [Default: %(default)s]""")
+P_call.add_argument("--ploidy", type=int, default=2,
+ help="Ploidy of the sample cells. [Default: %(default)d]")
+P_call.add_argument("--purity", type=float,
+ help="Estimated tumor cell fraction, a.k.a. purity or cellularity.")
+P_call.add_argument('-v', '--vcf',
+ help="""VCF file name containing variants for assigning allele
+ frequencies and copy number.""")
+P_call.add_argument("-g", "--gender",
+ choices=('m', 'male', 'Male', 'f', 'female', 'Female'),
+ help="""Specify the sample's gender as male or female. (Otherwise
+ guessed from chrX copy number).""")
+P_call.add_argument('-y', '--male-reference', action='store_true',
+ help="""Was a male reference used? If so, expect half ploidy on
+ chrX and chrY; otherwise, only chrY has half ploidy. In CNVkit,
+ if a male reference was used, the "neutral" copy number (ploidy)
+ of chrX is 1; chrY is haploid for either gender reference.""")
+P_call.add_argument('-o', '--output',
+ help="Output table file name (CNR-like table of segments, .cns).")
+P_call.set_defaults(func=_cmd_call)
+
+
+# _____________________________________________________________________________
+# Plots and graphics
+
+# diagram ---------------------------------------------------------------------
+
+def _cmd_diagram(args):
+ """Draw copy number (log2 coverages, CBS calls) on chromosomes as a diagram.
+
+ If both the raw probes and segments are given, show them side-by-side on
+ each chromosome (segments on the left side, probes on the right side).
+ """
+ from cnvlib import diagram
+ cnarr = _CNA.read(args.filename) if args.filename else None
+ segarr = _CNA.read(args.segment) if args.segment else None
+ outfname = diagram.create_diagram(cnarr, segarr, args.threshold,
+ args.min_probes, args.output,
+ args.male_reference)
+ logging.info("Wrote %s", outfname)
+
+
+P_diagram = AP_subparsers.add_parser('diagram', help=_cmd_diagram.__doc__)
+P_diagram.add_argument('filename', nargs='?',
+ help="""Processed coverage data file (*.cnr), the output of the
+ 'fix' sub-command.""")
+P_diagram.add_argument('-s', '--segment',
+ help="Segmentation calls (.cns), the output of the 'segment' command.")
+P_diagram.add_argument('-t', '--threshold', type=float, default=0.5,
+ help="""Copy number change threshold to label genes.
+ [Default: %(default)s]""")
+P_diagram.add_argument('-m', '--min-probes', type=int, default=3,
+ help="""Minimum number of covered probes to label a gene.
+ [Default: %(default)d]""")
+P_diagram.add_argument('-y', '--male-reference', action='store_true',
+ help="""Assume inputs are already corrected against a male
+ reference (i.e. female samples will have +1 log-CNR of
+ chrX; otherwise male samples would have -1 chrX).""")
+P_diagram.add_argument('-o', '--output',
+ help="Output PDF file name.")
+P_diagram.set_defaults(func=_cmd_diagram)
+
+
+# scatter ---------------------------------------------------------------------
+
+def _cmd_scatter(args):
+ """Plot probe log2 coverages and segmentation calls together."""
+ cnarr = _CNA.read(args.filename, args.sample_id
+ ) if args.filename else None
+ segarr = _CNA.read(args.segment
+ ) if args.segment else None
+ if not args.sample_id and (cnarr or segarr):
+ args.sample_id = (cnarr or segarr).sample_id
+ varr = _VA.read_vcf(args.vcf, args.sample_id, args.normal_id,
+ args.min_variant_depth, skip_hom=True, skip_somatic=True
+ ) if args.vcf else None
+
+ if args.range_list:
+ with PdfPages(args.output) as pdf_out:
+ for chrom, start, end in _RA.read(args.range_list).coords():
+ region = "{}:{}-{}".format(chrom, start, end)
+ do_scatter(cnarr, segarr, varr, region, False,
+ args.background_marker, args.trend,
+ args.width, args.y_min, args.y_max)
+ pyplot.title(region)
+ pdf_out.savefig()
+ pyplot.close()
+ else:
+ do_scatter(cnarr, segarr, varr, args.chromosome, args.gene,
+ args.background_marker, args.trend, args.width,
+ args.y_min, args.y_max)
+ if args.output:
+ pyplot.savefig(args.output, format='pdf', bbox_inches="tight")
+ logging.info("Wrote %s", args.output)
+ else:
+ pyplot.show()
+
+
+def do_scatter(cnarr, segments=None, variants=None,
+ show_range=None, show_gene=None,
+ background_marker=None, do_trend=False, window_width=1e6,
+ y_min=None, y_max=None, title=None):
+ """Plot probe log2 coverages and CBS calls together.
+
+ show_gene: name of gene to highligh
+ show_range: chromosome name or coordinate string like "chr1:20-30"
+ """
+ if title is None:
+ title = (cnarr or segments or variants).sample_id
+
+ if not show_gene and not show_range:
+ # Plot all chromosomes, concatenated on one plot
+ PAD = 1e7
+ if (cnarr or segments) and variants:
+ # Lay out top 3/5 for the CN scatter, bottom 2/5 for SNP plot
+ axgrid = pyplot.GridSpec(5, 1, hspace=.85)
+ axis = pyplot.subplot(axgrid[:3])
+ axis2 = pyplot.subplot(axgrid[3:], sharex=axis)
+ # Place chromosome labels between the CNR and SNP plots
+ axis2.tick_params(labelbottom=False)
+ chrom_sizes = plots.chromosome_sizes(cnarr or segments)
+ plots.snv_on_genome(axis2, variants, chrom_sizes, segments,
+ do_trend, PAD)
+ else:
+ _fig, axis = pyplot.subplots()
+ if cnarr or segments:
+ axis.set_title(title)
+ plots.cnv_on_genome(axis, cnarr, segments, PAD, do_trend, y_min, y_max)
+ else:
+ axis.set_title("Variant allele frequencies: %s" % title)
+ chrom_sizes = collections.OrderedDict(
+ (chrom, subarr["end"].max())
+ for chrom, subarr in variants.by_chromosome())
+ plots.snv_on_genome(axis, variants, chrom_sizes, segments, do_trend,
+ PAD)
+
+ else:
+ # Plot a specified region on one chromosome
+ # -r \ -g | None | Some
+ # None | genome| genes w/ auto window
+ # chr | chr | genes w/ no window *
+ # chr:s-e | window| genes w/ given window
+ chrom, start, end = plots.unpack_range(show_range)
+ window_coords = ()
+ genes = []
+ if show_gene:
+ gene_names = show_gene.split(',')
+ # Scan for probes matching the specified gene
+ gene_coords = plots.gene_coords_by_name(cnarr, gene_names)
+ if not len(gene_coords) == 1:
+ raise ValueError("Genes %s are split across chromosomes %s"
+ % (show_gene, gene_coords.keys()))
+ g_chrom, genes = gene_coords.popitem()
+ if chrom:
+ # Confirm that the selected chromosomes match
+ core.assert_equal("Chromosome also selected by region (-r)"
+ "does not match",
+ **{"chromosome": chrom,
+ "gene(s)": g_chrom})
+ else:
+ chrom = g_chrom
+ # Set the display window to the selected genes +/- a margin
+ genes.sort()
+ window_coords = (max(0, genes[0][0] - window_width),
+ genes[-1][1] + window_width)
+
+ if start is not None or end is not None:
+ # Default selection endpoint to the maximum chromosome position
+ if not end:
+ end = (cnarr or segments or variants
+ ).select(chromosome=chrom).end.iat[-1]
+ if window_coords:
+ # Genes were specified, & window was set around them
+ if start > window_coords[0] or end < window_coords[1]:
+ raise ValueError("Selected gene range " + chrom +
+ (":%d-%d" % window_coords) +
+ " is outside specified range " +
+ show_range)
+ window_coords = (max(0, start - window_width), end + window_width)
+ if cnarr and not genes:
+ genes = plots.gene_coords_by_range(cnarr, chrom,
+ start, end)[chrom]
+ if not genes and window_width > (end - start) / 10.0:
+ # No genes in the selected region, so highlight the region
+ # itself (unless the selection is ~entire displayed window)
+ logging.info("No genes found in selection; will show the "
+ "selected range itself instead")
+ genes = [(start, end, "Selection")]
+ elif show_range and window_coords:
+ # Specified range is only chrom, no start-end
+ # Reset window around selected genes to show the whole chromosome
+ window_coords = ()
+
+ # Prune plotted elements to the selected region
+ sel_probes = (cnarr.in_range(chrom, *window_coords)
+ if cnarr else _CNA([]))
+ sel_seg = (segments.in_range(chrom, *window_coords, mode='trim')
+ if segments else _CNA([]))
+
+ logging.info("Showing %d probes and %d selected genes in range %s",
+ len(sel_probes), len(genes),
+ (chrom + ":%d-%d" % window_coords if window_coords
+ else chrom))
+
+ # Similarly for SNV allele freqs, if given
+ if (cnarr or segments) and variants:
+ # Lay out top 3/5 for the CN scatter, bottom 2/5 for SNP plot
+ axgrid = pyplot.GridSpec(5, 1, hspace=.5)
+ axis = pyplot.subplot(axgrid[:3])
+ axis2 = pyplot.subplot(axgrid[3:], sharex=axis)
+ # Plot allele freqs for only the selected region
+ sel_snvs = variants.in_range(chrom, *window_coords)
+ plots.snv_on_chromosome(axis2, sel_snvs, sel_seg, genes,
+ do_trend, # do_boost,
+ )
+ elif variants:
+ # XXX tangle
+ # nb: don't do the last call to cnv_on_chromosome
+ _fig, axis = pyplot.subplots()
+ sel_snvs = variants.in_range(chrom, *window_coords)
+ plots.snv_on_chromosome(axis, sel_snvs, sel_seg, genes,
+ do_trend, # do_boost,
+ )
+ return
+
+ else:
+ _fig, axis = pyplot.subplots()
+ axis.set_xlabel("Position (Mb)")
+
+ # Plot CNVs
+ axis.set_title("%s %s" % (title, chrom))
+ plots.cnv_on_chromosome(axis, sel_probes, sel_seg, genes,
+ background_marker=background_marker,
+ do_trend=do_trend, y_min=y_min, y_max=y_max)
+
+
+P_scatter = AP_subparsers.add_parser('scatter', help=_cmd_scatter.__doc__)
+P_scatter.add_argument('filename', nargs="?",
+ help="""Processed bin-level copy ratios (*.cnr), the output
+ of the 'fix' sub-command.""")
+P_scatter.add_argument('-s', '--segment',
+ help="Segmentation calls (.cns), the output of the 'segment' command.")
+P_scatter.add_argument('-c', '--chromosome',
+ help="""Chromosome (e.g. 'chr1') or chromosomal range (e.g.
+ 'chr1:2333000-2444000') to display. If a range is given,
+ all targeted genes in this range will be shown, unless
+ '--gene'/'-g' is already given.""")
+P_scatter.add_argument('-g', '--gene',
+ help="Name of gene or genes (comma-separated) to display.")
+P_scatter.add_argument('-l', '--range-list',
+ help="""File listing the chromosomal ranges to display, as BED, interval
+ list or "chr:start-end" text. Creates focal plots similar to
+ -c/--chromosome for each listed region, combined into a
+ multi-page PDF. The output filename must also be
+ specified (-o/--output).""")
+P_scatter.add_argument("-i", "--sample-id",
+ help="""Specify the name of the sample in the VCF to use for b-allele
+ frequency extraction and to show in plot title.""")
+P_scatter.add_argument("-n", "--normal-id",
+ help="Corresponding normal sample ID in the input VCF.")
+P_scatter.add_argument('-b', '--background-marker', default=None,
+ help="""Plot antitargets with this symbol, in zoomed/selected regions.
+ [Default: same as targets]""")
+P_scatter.add_argument('-t', '--trend', action='store_true',
+ help="Draw a smoothed local trendline on the scatter plot.")
+P_scatter.add_argument('-v', '--vcf',
+ help="""VCF file name containing variants to plot for SNV allele
+ frequencies.""")
+P_scatter.add_argument('-m', '--min-variant-depth', type=int, default=20,
+ help="""Minimum read depth for a SNV to be displayed in the b-allele
+ frequency plot. [Default: %(default)s]""")
+P_scatter.add_argument('-w', '--width', type=float, default=1e6,
+ help="""Width of margin to show around the selected gene or region
+ on the chromosome (use with --gene or --region).
+ [Default: %(default)d]""")
+P_scatter.add_argument('--y-min', type=float, help="""y-axis lower limit.""")
+P_scatter.add_argument('--y-max', type=float, help="""y-axis upper limit.""")
+P_scatter.add_argument('-o', '--output',
+ help="Output table file name.")
+P_scatter.set_defaults(func=_cmd_scatter)
+
+
+# loh -------------------------------------------------------------------------
+
+def _cmd_loh(args):
+ """[DEPRECATED] Plot allelic frequencies at each variant position in a VCF file.
+
+ Divergence from 0.5 indicates loss of heterozygosity in a tumor sample.
+
+ Instead, use the command "scatter -v".
+ """
+ variants = _VA.read_vcf(args.variants, args.sample_id, args.normal_id,
+ args.min_depth, skip_hom=True, skip_somatic=True)
+ segments = _CNA.read(args.segment) if args.segment else None
+ _fig, axis = pyplot.subplots()
+ axis.set_title("Variant allele frequencies: %s" % variants.sample_id)
+ chrom_sizes = collections.OrderedDict(
+ (chrom, subarr["end"].max())
+ for chrom, subarr in variants.by_chromosome())
+ PAD = 2e7
+ plots.snv_on_genome(axis, variants, chrom_sizes, segments, args.trend, PAD)
+ if args.output:
+ pyplot.savefig(args.output, format='pdf', bbox_inches="tight")
+ else:
+ pyplot.show()
+
+
+P_loh = AP_subparsers.add_parser('loh', help=_cmd_loh.__doc__)
+P_loh.add_argument('variants',
+ help="Sample variants in VCF format.")
+P_loh.add_argument('-s', '--segment',
+ help="Segmentation calls (.cns), the output of the 'segment' command.")
+P_loh.add_argument('-m', '--min-depth', type=int, default=20,
+ help="""Minimum read depth for a variant to be displayed.
+ [Default: %(default)s]""")
+P_loh.add_argument("-i", "--sample-id",
+ help="Sample name to use for LOH calculations from the input VCF.")
+P_loh.add_argument("-n", "--normal-id",
+ help="Corresponding normal sample ID in the input VCF.")
+P_loh.add_argument('-t', '--trend', action='store_true',
+ help="Draw a smoothed local trendline on the scatter plot.")
+P_loh.add_argument('-o', '--output',
+ help="Output PDF file name.")
+P_loh.set_defaults(func=_cmd_loh)
+
+
+# heatmap ---------------------------------------------------------------------
+
+def _cmd_heatmap(args):
+ """Plot copy number for multiple samples as a heatmap."""
+ cnarrs = list(map(_CNA.read, args.filenames))
+ do_heatmap(cnarrs, args.chromosome, args.desaturate)
+ if args.output:
+ pyplot.savefig(args.output, format='pdf', bbox_inches="tight")
+ logging.info("Wrote %s", args.output)
+ else:
+ pyplot.show()
+
+
+def do_heatmap(cnarrs, show_range=None, do_desaturate=False):
+ """Plot copy number for multiple samples as a heatmap."""
+ from matplotlib.collections import BrokenBarHCollection
+
+ _fig, axis = pyplot.subplots()
+
+ # List sample names on the y-axis
+ axis.set_yticks([i + 0.5 for i in range(len(cnarrs))])
+ axis.set_yticklabels([c.sample_id for c in cnarrs])
+ axis.set_ylim(0, len(cnarrs))
+ axis.invert_yaxis()
+ axis.set_ylabel("Samples")
+ axis.set_axis_bgcolor('#DDDDDD')
+
+ r_chrom, r_start, r_end = plots.unpack_range(show_range)
+ if r_start is not None or r_end is not None:
+ logging.info("Showing log2 ratios in range %s:%d-%s",
+ r_chrom, r_start, r_end or '*')
+ elif r_chrom:
+ logging.info("Showing log2 ratios on chromosome %s", r_chrom)
+
+ # Closes over do_desaturate
+ def cna2df(cna):
+ """Extract a dataframe of plotting points from a CopyNumArray."""
+ points = cna.data.loc[:, ["start", "end"]]
+ points["color"] = cna.log2.apply(plots.cvg2rgb, args=(do_desaturate,))
+ return points
+
+ # Group each file's probes/segments by chromosome
+ sample_data = [collections.defaultdict(list) for _c in cnarrs]
+ # Calculate the size (max endpoint value) of each chromosome
+ chrom_sizes = collections.OrderedDict()
+ for i, cnarr in enumerate(cnarrs):
+ if r_chrom:
+ subcna = cnarr.in_range(r_chrom, r_start, r_end, mode="trim")
+ sample_data[i][r_chrom] = cna2df(subcna)
+ chrom_sizes[r_chrom] = max(subcna.end.iat[-1] if subcna else 0,
+ chrom_sizes.get(r_chrom, 0))
+ else:
+ for chrom, subcna in cnarr.by_chromosome():
+ sample_data[i][chrom] = cna2df(subcna)
+ chrom_sizes[chrom] = max(subcna.end.iat[-1] if subcna else 0,
+ chrom_sizes.get(r_chrom, 0))
+
+ # Closes over axis
+ def plot_sample_chrom(i, sample):
+ """Draw the given coordinates and colors as a horizontal series."""
+ xranges = [(start, end - start)
+ for start, end in zip(sample.start, sample.end)]
+ bars = BrokenBarHCollection(xranges, (i, i+1),
+ edgecolors="none",
+ facecolors=sample["color"])
+ axis.add_collection(bars)
+
+ if show_range:
+ # Lay out only the selected chromosome
+ # Set x-axis the chromosomal positions (in Mb), title as the selection
+ axis.set_xlim((r_start or 0) * plots.MB,
+ (r_end or chrom_sizes[r_chrom]) * plots.MB)
+ axis.set_title(show_range)
+ axis.set_xlabel("Position (Mb)")
+ axis.tick_params(which='both', direction='out')
+ axis.get_xaxis().tick_bottom()
+ axis.get_yaxis().tick_left()
+ # Plot the individual probe/segment coverages
+ for i, sample in enumerate(sample_data):
+ crow = sample[r_chrom]
+ crow["start"] *= plots.MB
+ crow["end"] *= plots.MB
+ plot_sample_chrom(i, crow)
+
+ else:
+ # Lay out chromosome dividers and x-axis labels
+ # (Just enough padding to avoid overlap with the divider line)
+ chrom_offsets = plots.plot_x_dividers(axis, chrom_sizes, 1)
+ # Plot the individual probe/segment coverages
+ for i, sample in enumerate(sample_data):
+ for chrom, curr_offset in iteritems(chrom_offsets):
+ crow = sample[chrom]
+ crow["start"] += curr_offset
+ crow["end"] += curr_offset
+ plot_sample_chrom(i, crow)
+
+ return axis
+
+
+P_heatmap = AP_subparsers.add_parser('heatmap', help=_cmd_heatmap.__doc__)
+P_heatmap.add_argument('filenames', nargs='+',
+ help="Sample coverages as raw probes (.cnr) or segments (.cns).")
+P_heatmap.add_argument('-c', '--chromosome',
+ help="""Chromosome (e.g. 'chr1') or chromosomal range (e.g.
+ 'chr1:2333000-2444000') to display. If a range is given,
+ all targeted genes in this range will be shown, unless
+ '--gene'/'-g' is already given.""")
+# P_heatmap.add_argument('-g', '--gene',
+# help="Name of gene to display.")
+P_heatmap.add_argument('-d', '--desaturate', action='store_true',
+ help="Tweak color saturation to focus on significant changes.")
+P_heatmap.add_argument('-o', '--output',
+ help="Output PDF file name.")
+P_heatmap.set_defaults(func=_cmd_heatmap)
+
+
+# _____________________________________________________________________________
+# Tabular outputs
+
+# breaks ----------------------------------------------------------------------
+
+def _cmd_breaks(args):
+ """List the targeted genes in which a copy number breakpoint occurs."""
+ cnarr = _CNA.read(args.filename)
+ segarr = _CNA.read(args.segment)
+ bpoints = do_breaks(cnarr, segarr, args.min_probes)
+ logging.info("Found %d gene breakpoints", len(bpoints))
+ core.write_tsv(args.output, bpoints,
+ colnames=['Gene', 'Chrom.', 'Location', 'Change',
+ 'ProbesLeft', 'ProbesRight'])
+
+
+def do_breaks(probes, segments, min_probes=1):
+ """List the targeted genes in which a copy number breakpoint occurs."""
+ intervals = reports.get_gene_intervals(probes)
+ bpoints = reports.get_breakpoints(intervals, segments, min_probes)
+ return bpoints
+
+
+P_breaks = AP_subparsers.add_parser('breaks', help=_cmd_breaks.__doc__)
+P_breaks.add_argument('filename',
+ help="""Processed sample coverage data file (*.cnr), the output
+ of the 'fix' sub-command.""")
+P_breaks.add_argument('segment',
+ help="Segmentation calls (.cns), the output of the 'segment' command).")
+P_breaks.add_argument('-m', '--min-probes', type=int, default=1,
+ help="""Minimum number of within-gene probes on both sides of a
+ breakpoint to report it. [Default: %(default)d]""")
+P_breaks.add_argument('-o', '--output',
+ help="Output table file name.")
+P_breaks.set_defaults(func=_cmd_breaks)
+
+
+# gainloss --------------------------------------------------------------------
+
+def _cmd_gainloss(args):
+ """Identify targeted genes with copy number gain or loss."""
+ pset = _CNA.read(args.filename)
+ segs = _CNA.read(args.segment) if args.segment else None
+ gainloss = do_gainloss(pset, segs, args.male_reference, args.threshold,
+ args.min_probes, args.drop_low_coverage)
+ logging.info("Found %d gene-level gains and losses", len(gainloss))
+ core.write_tsv(args.output, gainloss,
+ colnames=['Gene', 'Chrom.', 'Start', 'End', 'Log2Ratio',
+ 'Probes'])
+
+
+def do_gainloss(probes, segments=None, male_reference=False, threshold=0.2,
+ min_probes=3, skip_low=False):
+ """Identify targeted genes with copy number gain or loss."""
+ probes = probes.shift_xx(male_reference)
+ if segments:
+ segments = segments.shift_xx(male_reference)
+ gainloss = reports.gainloss_by_segment(probes, segments, threshold,
+ skip_low)
+ else:
+ gainloss = reports.gainloss_by_gene(probes, threshold, skip_low)
+ return [row for row in gainloss if row[5] >= min_probes]
+
+
+P_gainloss = AP_subparsers.add_parser('gainloss', help=_cmd_gainloss.__doc__)
+P_gainloss.add_argument('filename',
+ help="""Processed sample coverage data file (*.cnr), the output
+ of the 'fix' sub-command.""")
+P_gainloss.add_argument('-s', '--segment',
+ help="Segmentation calls (.cns), the output of the 'segment' command).")
+P_gainloss.add_argument('-t', '--threshold', type=float, default=0.2,
+ help="""Copy number change threshold to report a gene gain/loss.
+ [Default: %(default)s]""")
+P_gainloss.add_argument('-m', '--min-probes', type=int, default=3,
+ help="""Minimum number of covered probes to report a gain/loss.
+ [Default: %(default)d]""")
+P_gainloss.add_argument("--drop-low-coverage", action='store_true',
+ help="""Drop very-low-coverage bins before segmentation to avoid
+ false-positive deletions in poor-quality tumor samples.""")
+P_gainloss.add_argument('-y', '--male-reference', action='store_true',
+ help="""Assume inputs are already corrected against a male
+ reference (i.e. female samples will have +1 log-coverage of
+ chrX; otherwise male samples would have -1 chrX).""")
+P_gainloss.add_argument('-o', '--output',
+ help="Output table file name.")
+P_gainloss.set_defaults(func=_cmd_gainloss)
+
+
+# gender ----------------------------------------------------------------------
+
+def _cmd_gender(args):
+ """Guess samples' gender from the relative coverage of chromosome X."""
+ outrows = []
+ for fname in args.targets:
+ rel_chrx_cvg = _CNA.read(fname).get_relative_chrx_cvg()
+ if args.male_reference:
+ is_xx = (rel_chrx_cvg >= 0.5)
+ else:
+ is_xx = (rel_chrx_cvg >= -0.5)
+ outrows.append((fname,
+ ("Female" if is_xx else "Male"),
+ "%s%.3g" % ('+' if rel_chrx_cvg > 0 else '',
+ rel_chrx_cvg)))
+ core.write_tsv(args.output, outrows)
+
+
+P_gender = AP_subparsers.add_parser('gender', help=_cmd_gender.__doc__)
+P_gender.add_argument('targets', nargs='+',
+ help="Copy number or copy ratio files (*.cnn, *.cnr).")
+P_gender.add_argument('-y', '--male-reference', action='store_true',
+ help="""Assume inputs are already normalized to a male reference
+ (i.e. female samples will have +1 log-coverage of chrX;
+ otherwise male samples would have -1 chrX).""")
+P_gender.add_argument('-o', '--output',
+ help="Output table file name.")
+P_gender.set_defaults(func=_cmd_gender)
+
+
+# metrics ---------------------------------------------------------------------
+
+def _cmd_metrics(args):
+ """Compute coverage deviations and other metrics for self-evaluation.
+ """
+ if (len(args.cnarrays) > 1 and len(args.segments) > 1 and
+ len(args.cnarrays) != len(args.segments)):
+ raise ValueError("Number of coverage/segment filenames given must be "
+ "equal, if more than 1 segment file is given.")
+
+ # Repeat a single segment file to match the number of coverage files
+ if len(args.cnarrays) > 1 and len(args.segments) == 1:
+ args.segments = [args.segments[0] for _i in range(len(args.cnarrays))]
+
+ # Calculate all metrics
+ outrows = []
+ for probes_fname, segs_fname in zip(args.cnarrays, args.segments):
+ cnarr = _CNA.read(probes_fname)
+ segments = _CNA.read(segs_fname)
+ values = metrics.ests_of_scale(cnarr.drop_low_coverage()
+ .residuals(segments))
+ outrows.append([core.rbase(probes_fname), len(segments)] +
+ ["%.7f" % val for val in values])
+
+ core.write_tsv(args.output, outrows,
+ colnames=("sample", "segments", "stdev", "mad", "iqr",
+ "bivar"))
+
+
+P_metrics = AP_subparsers.add_parser('metrics', help=_cmd_metrics.__doc__)
+P_metrics.add_argument('cnarrays', nargs='+',
+ help="""One or more bin-level coverage data files (*.cnn, *.cnr).""")
+P_metrics.add_argument('-s', '--segments', nargs='+', required=True,
+ help="""One or more segmentation data files (*.cns, output of the
+ 'segment' command). If more than one file is given, the number
+ must match the coverage data files, in which case the input
+ files will be paired together in the given order. Otherwise, the
+ same segments will be used for all coverage files.""")
+P_metrics.add_argument('-o', '--output',
+ help="Output table file name.")
+P_metrics.set_defaults(func=_cmd_metrics)
+
+
+# segmetrics ------------------------------------------------------------------
+
+def _cmd_segmetrics(args):
+ """Compute segment-level metrics from bin-level log2 ratios."""
+ if not 0.0 < args.alpha <= 1.0:
+ raise RuntimeError("alpha must be between 0 and 1.")
+
+ stats = {
+ 'stdev': np.std,
+ 'mad': metrics.median_absolute_deviation,
+ 'iqr': metrics.interquartile_range,
+ 'bivar': metrics.biweight_midvariance,
+ 'ci': lambda x: metrics.confidence_interval_bootstrap(x, args.alpha,
+ args.bootstrap),
+ 'pi': lambda x: metrics.prediction_interval(x, args.alpha),
+ }
+ if not any(getattr(args, name) for name in stats):
+ logging.info("No stats specified")
+ return
+
+ # Calculate all metrics
+ cnarr = _CNA.read(args.cnarray)
+ if args.drop_low_coverage:
+ cnarr = cnarr.drop_low_coverage()
+ segarr = _CNA.read(args.segments)
+ deviations = [segbins.log2 - segment.log2
+ for segment, segbins in cnarr.by_ranges(segarr)]
+ # Measures of spread
+ for statname in ("StDev", "MAD", "IQR", "BiVar"):
+ option = statname.lower()
+ if getattr(args, option):
+ func = stats[option]
+ segarr[statname] = np.asfarray([func(d) for d in deviations])
+
+ # Interval calculations
+ if args.ci:
+ segarr["CI_lo"], segarr["CI_hi"] = _segmetric_interval(segarr, cnarr,
+ stats['ci'])
+ if args.pi:
+ segarr["PI_lo"], segarr["PI_hi"] = _segmetric_interval(segarr, cnarr,
+ stats['pi'])
+
+ segarr.write(args.output or segarr.sample_id + ".segmetrics.cns")
+
+
+def _segmetric_interval(segarr, cnarr, func):
+ """Compute a stat that yields intervals (low & high values)"""
+ out_vals_lo = np.repeat(np.nan, len(segarr))
+ out_vals_hi = np.repeat(np.nan, len(segarr))
+ for i, (_segment, bins) in enumerate(cnarr.by_ranges(segarr)):
+ if len(bins):
+ out_vals_lo[i], out_vals_hi[i] = func(bins)
+ return out_vals_lo, out_vals_hi
+
+
+P_segmetrics = AP_subparsers.add_parser('segmetrics', help=_cmd_segmetrics.__doc__)
+P_segmetrics.add_argument('cnarray',
+ help="""Bin-level copy ratio data file (*.cnn, *.cnr).""")
+P_segmetrics.add_argument('-s', '--segments', required=True,
+ help="Segmentation data file (*.cns, output of the 'segment' command).")
+P_segmetrics.add_argument("--drop-low-coverage", action='store_true',
+ help="""Drop very-low-coverage bins before segmentation to avoid
+ false-positive deletions in poor-quality tumor samples.""")
+P_segmetrics.add_argument('-o', '--output',
+ help="Output table file name.")
+
+P_segmetrics_stats = P_segmetrics.add_argument_group(
+ "Statistics available")
+P_segmetrics_stats.add_argument('--stdev', action='store_true',
+ help="Standard deviation.")
+P_segmetrics_stats.add_argument('--mad', action='store_true',
+ help="Median absolute deviation (standardized).")
+P_segmetrics_stats.add_argument('--iqr', action='store_true',
+ help="Inter-quartile range.")
+P_segmetrics_stats.add_argument('--bivar', action='store_true',
+ help="Tukey's biweight midvariance.")
+P_segmetrics_stats.add_argument('--ci', action='store_true',
+ help="Confidence interval (by bootstrap).")
+P_segmetrics_stats.add_argument('--pi', action='store_true',
+ help="Prediction interval.")
+P_segmetrics_stats.add_argument('-a', '--alpha', type=float, default=.05,
+ help="""Level to estimate confidence and prediction intervals;
+ use with --ci and --pi. [Default: %(default)s]""")
+P_segmetrics_stats.add_argument('-b', '--bootstrap', type=int, default=100,
+ help="""Number of bootstrap iterations to estimate confidence interval;
+ use with --ci. [Default: %(default)d]""")
+P_segmetrics.set_defaults(func=_cmd_segmetrics)
+
+
+# _____________________________________________________________________________
+# Other I/O and compatibility
+
+# import-picard ---------------------------------------------------------------
+
+def _cmd_import_picard(args):
+ """Convert Picard CalculateHsMetrics tabular output to CNVkit .cnn files.
+
+ The input file is generated by the PER_TARGET_COVERAGE option in the
+ CalculateHsMetrics script in Picard tools.
+ """
+ for fname in importers.find_picard_files(args.targets):
+ cnarr = importers.import_picard_pertargetcoverage(fname)
+ outfname = os.path.basename(fname)[:-4] + '.cnn'
+ if args.output_dir:
+ if not os.path.isdir(args.output_dir):
+ os.mkdir(args.output_dir)
+ logging.info("Created directory %s", args.output_dir)
+ outfname = os.path.join(args.output_dir, outfname)
+ cnarr.write(outfname)
+
+
+P_import_picard = AP_subparsers.add_parser('import-picard',
+ help=_cmd_import_picard.__doc__)
+P_import_picard.add_argument('targets', nargs='*', default=['.'],
+ help="""Sample coverage .csv files (target and antitarget), or the
+ directory that contains them.""")
+P_import_picard.add_argument('-d', '--output-dir', default='.',
+ help="Output directory name.")
+P_import_picard.set_defaults(func=_cmd_import_picard)
+
+
+# import-seg ------------------------------------------------------------------
+
+def _cmd_import_seg(args):
+ """Convert a SEG file to CNVkit .cns files."""
+ if args.chromosomes:
+ if args.chromosomes == 'human':
+ chrom_names = {'23': 'X', '24': 'Y', '25': 'M'}
+ else:
+ chrom_names = dict(kv.split(':')
+ for kv in args.chromosomes.split(','))
+ else:
+ chrom_names = args.chromosomes
+
+ for segset in importers.import_seg(args.segfile, chrom_names, args.prefix,
+ args.from_log10):
+ segset.write(os.path.join(args.output_dir, segset.sample_id + '.cns'))
+
+
+P_import_seg = AP_subparsers.add_parser('import-seg',
+ help=_cmd_import_seg.__doc__)
+P_import_seg.add_argument('segfile',
+ help="""Input file in SEG format. May contain multiple samples.""")
+P_import_seg.add_argument('-c', '--chromosomes',
+ help="""Mapping of chromosome indexes to names. Syntax:
+ "from1:to1,from2:to2". Or use "human" for the preset:
+ "23:X,24:Y,25:M".""")
+P_import_seg.add_argument('-p', '--prefix',
+ help="""Prefix to add to chromosome names (e.g 'chr' to rename '8' in
+ the SEG file to 'chr8' in the output).""")
+P_import_seg.add_argument('--from-log10', action='store_true',
+ help="Convert base-10 logarithm values in the input to base-2 logs.")
+P_import_seg.add_argument('-d', '--output-dir', default='.',
+ help="Output directory name.")
+P_import_seg.set_defaults(func=_cmd_import_seg)
+
+
+# import-theta ---------------------------------------------------------------
+
+def _cmd_import_theta(args):
+ """Convert THetA output to a BED-like, CNVkit-like tabular format.
+
+ Equivalently, use the THetA results file to convert CNVkit .cns segments to
+ integer copy number calls.
+ """
+ tumor_segs = _CNA.read(args.tumor_cns)
+ for i, new_cns in enumerate(do_import_theta(tumor_segs, args.theta_results,
+ args.ploidy)):
+ new_cns.write(os.path.join(args.output_dir,
+ "%s-%d.cns" % (tumor_segs.sample_id, i + 1)))
+
+
+def do_import_theta(segarr, theta_results_fname, ploidy=2):
+ theta = importers.parse_theta_results(theta_results_fname)
+ for copies in theta['C']:
+ # Drop any segments where the C value is None
+ mask_drop = np.array([c is None for c in copies], dtype='bool')
+ segarr = segarr[~mask_drop].copy()
+ ok_copies = np.array([c for c in copies if c is not None], dtype='int')
+ # Replace remaining segment values with these integers
+ segarr["cn"] = ok_copies.copy()
+ ok_copies[ok_copies == 0] = 0.5
+ segarr["log2"] = np.log2(ok_copies / ploidy)
+ segarr.sort_columns()
+ yield segarr
+
+
+P_import_theta = AP_subparsers.add_parser('import-theta',
+ help=_cmd_import_theta.__doc__)
+P_import_theta.add_argument("tumor_cns")
+P_import_theta.add_argument("theta_results")
+P_import_theta.add_argument("--ploidy", type=int, default=2,
+ help="Ploidy of normal cells. [Default: %(default)d]")
+P_import_theta.add_argument('-d', '--output-dir', default='.',
+ help="Output directory name.")
+P_import_theta.set_defaults(func=_cmd_import_theta)
+
+
+# export ----------------------------------------------------------------------
+
+P_export = AP_subparsers.add_parser('export',
+ help="""Convert CNVkit output files to another format.""")
+P_export_subparsers = P_export.add_subparsers(
+ help="Export formats (use with -h for more info).")
+
+
+# BED special case: multiple samples's segments, like SEG
+def _cmd_export_bed(args):
+ """Convert segments to BED format.
+
+ Input is a segmentation file (.cns) where, preferably, log2 ratios have
+ already been adjusted to integer absolute values using the 'call' command.
+ """
+ if args.show_all and args.show == "ploidy":
+ # Until this option is removed, let it override the default
+ logging.warn("Option '--show-all' is deprecated; "
+ "use '--show all' instead.")
+ args.show = "all"
+ bed_tables = []
+ for segfname in args.segments:
+ segments = _CNA.read(segfname)
+ # ENH: args.gender as a comma-separated list of genders
+ is_sample_female = verify_gender_arg(segments, args.gender,
+ args.male_reference)
+ tbl = export.export_bed(segments, args.ploidy,
+ args.male_reference, is_sample_female,
+ args.sample_id or segments.sample_id,
+ args.show)
+ bed_tables.append(tbl)
+ table = pd.concat(bed_tables)
+ core.write_dataframe(args.output, table, header=False)
+
+P_export_bed = P_export_subparsers.add_parser('bed',
+ help=_cmd_export_bed.__doc__)
+P_export_bed.add_argument('segments', nargs='+',
+ help="""Segmented copy ratio data files (*.cns), the output of the
+ 'segment' or 'call' sub-commands.""")
+P_export_bed.add_argument("-i", "--sample-id", metavar="LABEL",
+ help="""Identifier to write in the 4th column of the BED file.
+ [Default: use the sample ID, taken from the file name]""")
+P_export_bed.add_argument("--ploidy", type=int, default=2,
+ help="Ploidy of the sample cells. [Default: %(default)d]")
+P_export_bed.add_argument("-g", "--gender",
+ choices=('m', 'male', 'Male', 'f', 'female', 'Female'),
+ help="""Specify the sample's gender as male or female. (Otherwise
+ guessed from chrX copy number).""")
+P_export_bed.add_argument("--show",
+ choices=('ploidy', 'variant', 'all'), default="ploidy",
+ help="""Which segmented regions to show:
+ 'all' = all segment regions;
+ 'variant' = CNA regions with non-neutral copy number;
+ 'ploidy' = CNA regions with non-default ploidy.
+ [Default: %(default)s]""")
+P_export_bed.add_argument("--show-all", action="store_true",
+ help="""Write all segmented regions.
+ [DEPRECATED; use "--show all" instead]""")
+P_export_bed.add_argument("-y", "--male-reference", action="store_true",
+ help="""Was a male reference used? If so, expect half ploidy on
+ chrX and chrY; otherwise, only chrY has half ploidy. In CNVkit,
+ if a male reference was used, the "neutral" copy number (ploidy)
+ of chrX is 1; chrY is haploid for either gender reference.""")
+P_export_bed.add_argument('-o', '--output', help="Output file name.")
+P_export_bed.set_defaults(func=_cmd_export_bed)
+
+
+# SEG special case: segment coords don't match across samples
+def _cmd_export_seg(args):
+ """Convert segments to SEG format.
+
+ Compatible with IGV and GenePattern.
+ """
+ table = export.export_seg(args.filenames)
+ core.write_dataframe(args.output, table)
+
+P_export_seg = P_export_subparsers.add_parser('seg',
+ help=_cmd_export_seg.__doc__)
+P_export_seg.add_argument('filenames', nargs='+',
+ help="""Segmented copy ratio data file(s) (*.cns), the output of the
+ 'segment' sub-command.""")
+P_export_seg.add_argument('-o', '--output', help="Output file name.")
+P_export_seg.set_defaults(func=_cmd_export_seg)
+
+
+# VCF special case: only 1 sample, for now
+def _cmd_export_vcf(args):
+ """Convert segments to VCF format.
+
+ Input is a segmentation file (.cns) where, preferably, log2 ratios have
+ already been adjusted to integer absolute values using the 'call' command.
+ """
+ segments = _CNA.read(args.segments)
+ is_sample_female = verify_gender_arg(segments,
+ args.gender,
+ args.male_reference)
+ header, body = export.export_vcf(segments, args.ploidy, args.male_reference,
+ is_sample_female, args.sample_id)
+ core.write_text(args.output, header, body)
+
+P_export_vcf = P_export_subparsers.add_parser('vcf',
+ help=_cmd_export_vcf.__doc__)
+P_export_vcf.add_argument('segments', #nargs='1',
+ help="""Segmented copy ratio data file (*.cns), the output of the
+ 'segment' or 'call' sub-commands.""")
+P_export_vcf.add_argument("-i", "--sample-id", metavar="LABEL",
+ help="""Sample name to write in the genotype field of the output VCF file.
+ [Default: use the sample ID, taken from the file name]""")
+P_export_vcf.add_argument("--ploidy", type=int, default=2,
+ help="Ploidy of the sample cells. [Default: %(default)d]")
+P_export_vcf.add_argument("-g", "--gender",
+ choices=('m', 'male', 'Male', 'f', 'female', 'Female'),
+ help="""Specify the sample's gender as male or female. (Otherwise
+ guessed from chrX copy number).""")
+P_export_vcf.add_argument("-y", "--male-reference", action="store_true",
+ help="""Was a male reference used? If so, expect half ploidy on
+ chrX and chrY; otherwise, only chrY has half ploidy. In CNVkit,
+ if a male reference was used, the "neutral" copy number (ploidy)
+ of chrX is 1; chrY is haploid for either gender reference.""")
+P_export_vcf.add_argument('-o', '--output', help="Output file name.")
+P_export_vcf.set_defaults(func=_cmd_export_vcf)
+
+
+# THetA special case: takes tumor .cns and normal .cnr or reference.cnn
+def _cmd_export_theta(args):
+ """Convert segments to THetA2 input file format (*.input)."""
+ tumor_cn = _CNA.read(args.tumor_segment)
+ # Handle deprecated positional reference
+ if args.normal_reference:
+ logging.warn("Second positional argument normal_reference is "
+ "deprecated; use --reference instead.")
+ if not args.reference:
+ args.reference = args.normal_reference
+ normal_cn = (_CNA.read(args.reference) if args.reference else None)
+ table = export.export_theta(tumor_cn, normal_cn)
+ if not args.output:
+ args.output = tumor_cn.sample_id + ".interval_count"
+ table.to_csv(args.output, sep='\t', index=False)
+ logging.info("Wrote %s", args.output)
+ if args.vcf:
+ variants = _VA.read_vcf(args.vcf,
+ sample_id=args.sample_id or tumor_cn.sample_id,
+ normal_id=args.normal_id, min_depth=args.min_depth,
+ skip_somatic=True, skip_hom=False)
+ tumor_snps, normal_snps = export.export_theta_snps(variants)
+ for title, table in [("tumor", tumor_snps), ("normal", normal_snps)]:
+ out_fname = "{}.{}.snp_formatted.txt".format(tumor_cn.sample_id, title)
+ table.to_csv(out_fname, sep='\t', index=False)
+ logging.info("Wrote %s", out_fname)
+
+P_export_theta = P_export_subparsers.add_parser('theta',
+ help=_cmd_export_theta.__doc__)
+P_export_theta.add_argument('tumor_segment',
+ help="""Tumor-sample segmentation file from CNVkit (.cns).""")
+P_export_theta.add_argument("normal_reference", nargs='?',
+ help="""Reference copy number profile (.cnn), or normal-sample bin-level
+ log2 copy ratios (.cnr). [DEPRECATED]""")
+P_export_theta.add_argument("-r", "--reference",
+ help="""Reference copy number profile (.cnn), or normal-sample bin-level
+ log2 copy ratios (.cnr). Use if the tumor_segment input file
+ does not contain a "weight" column.""")
+P_export_theta.add_argument("-v", "--vcf",
+ help="""VCF file containing SNVs observed in both the tumor and normal
+ samples. Tumor sample ID should match the `tumor_segment`
+ filename or be specified with -i/--sample-id.""")
+P_export_theta.add_argument("-i", "--sample-id",
+ help="""Specify the name of the tumor sample in the VCF (given with
+ -v/--vcf). [Default: taken the tumor_segment file name]""")
+P_export_theta.add_argument("-n", "--normal-id",
+ help="Corresponding normal sample ID in the input VCF.")
+P_export_theta.add_argument('-m', '--min-depth', type=int, default=20,
+ help="""Minimum read depth for a SNP in the VCF to be counted.
+ [Default: %(default)s]""")
+P_export_theta.add_argument('-o', '--output', help="Output file name.")
+P_export_theta.set_defaults(func=_cmd_export_theta)
+
+
+# Nexus "basic" special case: can only represent 1 sample
+def _cmd_export_nb(args):
+ """Convert bin-level log2 ratios to Nexus Copy Number "basic" format."""
+ table = export.export_nexus_basic(args.filename)
+ core.write_dataframe(args.output, table)
+
+P_export_nb = P_export_subparsers.add_parser('nexus-basic',
+ help=_cmd_export_nb.__doc__)
+P_export_nb.add_argument('filename',
+ help="""Log2 copy ratio data file (*.cnr), the output of the 'fix'
+ sub-command.""")
+P_export_nb.add_argument('-o', '--output', help="Output file name.")
+P_export_nb.set_defaults(func=_cmd_export_nb)
+
+
+# Nexus "Custom-OGT" special case: can only represent 1 sample
+def _cmd_export_nbo(args):
+ """Convert log2 ratios and b-allele freqs to Nexus "Custom-OGT" format."""
+ table = export.export_nexus_ogt(args.filename, args.vcf, args.sample_id,
+ args.min_variant_depth, args.min_weight)
+ core.write_dataframe(args.output, table)
+
+P_export_nbo = P_export_subparsers.add_parser('nexus-ogt',
+ help=_cmd_export_nbo.__doc__)
+P_export_nbo.add_argument('filename',
+ help="""Log2 copy ratio data file (*.cnr), the output of the 'fix'
+ sub-command.""")
+P_export_nbo.add_argument('vcf',
+ help="""VCF of SNVs for the same sample, to calculate b-allele
+ frequencies.""")
+P_export_nbo.add_argument("-i", "--sample-id",
+ help="""Specify the name of the sample in the VCF to use to extract
+ b-allele frequencies.""")
+P_export_nbo.add_argument('-m', '--min-variant-depth', type=int, default=20,
+ help="""Minimum read depth for a SNV to be included in the b-allele
+ frequency calculation. [Default: %(default)s]""")
+P_export_nbo.add_argument('-w', '--min-weight', type=float, default=0.0,
+ help="""Minimum weight (between 0 and 1) for a bin to be included in
+ the output. [Default: %(default)s]""")
+P_export_nbo.add_argument('-o', '--output', help="Output file name.")
+P_export_nbo.set_defaults(func=_cmd_export_nbo)
+
+
+# All else: export any number of .cnr or .cns files
+
+for fmt_key, fmt_descr in (
+ ('cdt', "Convert log2 ratios to CDT format. Compatible with Java TreeView."),
+ ('jtv', "Convert log2 ratios to Java TreeView's native format."),
+ # Not implemented yet:
+ # 'gct' (GenePattern).
+):
+ def _cmd_export_simple(args):
+ sample_ids = list(map(core.fbase, args.filenames))
+ table = export.merge_samples(args.filenames)
+ formatter = export.EXPORT_FORMATS[fmt_key]
+ outheader, outrows = formatter(sample_ids, table)
+ core.write_tsv(args.output, outrows, colnames=outheader)
+
+ P_export_simple = P_export_subparsers.add_parser(fmt_key, help=fmt_descr)
+ P_export_simple.add_argument('filenames', nargs='+',
+ help="""Log2 copy ratio data file(s) (*.cnr), the output of the
+ 'fix' sub-command.""")
+ P_export_simple.add_argument('-o', '--output', help="Output file name.")
+ P_export_simple.set_defaults(func=_cmd_export_simple)
+
+
+# version ---------------------------------------------------------------------
+
+def print_version(_args):
+ """Display this program's version."""
+ print(__version__)
+
+
+P_version = AP_subparsers.add_parser('version', help=print_version.__doc__)
+P_version.set_defaults(func=print_version)
+
+
+# _____________________________________________________________________________
+# Shim for command-line execution
+
+def parse_args(args=None):
+ """Parse the command line."""
+ return AP.parse_args(args=args)
diff --git a/cnvlib/core.py b/cnvlib/core.py
new file mode 100644
index 0000000..69a51a1
--- /dev/null
+++ b/cnvlib/core.py
@@ -0,0 +1,115 @@
+"""CNV utilities."""
+from __future__ import absolute_import, division, print_function
+import sys
+import os.path
+from itertools import takewhile
+
+from .ngfrills import safe_write
+
+# __________________________________________________________________________
+# I/O helpers
+
+def write_tsv(outfname, rows, colnames=None):
+ """Write rows, with optional column header, to tabular file."""
+ with safe_write(outfname or sys.stdout) as handle:
+ if colnames:
+ header = '\t'.join(colnames) + '\n'
+ handle.write(header)
+ handle.writelines('\t'.join(map(str, row)) + '\n'
+ for row in rows)
+
+
+def write_text(outfname, text, *more_texts):
+ """Write one or more strings (blocks of text) to a file."""
+ with safe_write(outfname or sys.stdout) as handle:
+ handle.write(text)
+ if more_texts:
+ for mtext in more_texts:
+ handle.write(mtext)
+
+
+def write_dataframe(outfname, dframe, header=True):
+ """Write a pandas.DataFrame to a tabular file."""
+ with safe_write(outfname or sys.stdout) as handle:
+ dframe.to_csv(handle, header=header,
+ index=False, sep='\t', float_format='%.6g')
+
+
+# __________________________________________________________________________
+# Sorting key functions
+
+def sorter_chrom(label):
+ """Create a sorting key from chromosome label.
+
+ Sort by integers first, then letters or strings. The prefix "chr"
+ (case-insensitive), if present, is stripped automatically for sorting.
+
+ E.g. chr1 < chr2 < chr10 < chrX < chrY < chrM
+ """
+ # Strip "chr" prefix
+ chrom = (label[3:] if label.lower().startswith('chr')
+ else label)
+ if chrom in ('X', 'Y'):
+ key = (1000, chrom)
+ else:
+ # Separate numeric and special chromosomes
+ nums = ''.join(takewhile(str.isdigit, chrom))
+ chars = chrom[len(nums):]
+ nums = int(nums) if nums else 0
+ if not chars:
+ key = (nums, '')
+ elif len(chars) == 1:
+ key = (2000 + nums, chars)
+ else:
+ key = (3000 + nums, chars)
+ return key
+
+
+def sorter_chrom_at(index):
+ """Create a sort key function that gets chromosome label at a list index."""
+ return lambda row: sorter_chrom(row[index])
+
+
+# __________________________________________________________________________
+# More helpers
+
+def assert_equal(msg, **values):
+ """Evaluate and compare two or more values for equality.
+
+ Sugar for a common assertion pattern. Saves re-evaluating (and retyping)
+ the same values for comparison and error reporting.
+
+ Example:
+
+ >>> assert_equal("Mismatch", expected=1, saw=len(['xx', 'yy']))
+ ...
+ ValueError: Mismatch: expected = 1, saw = 2
+
+ """
+ ok = True
+ key1, val1 = values.popitem()
+ msg += ": %s = %r" % (key1, val1)
+ for okey, oval in values.iteritems():
+ msg += ", %s = %r" % (okey, oval)
+ if oval != val1:
+ ok = False
+ if not ok:
+ raise ValueError(msg)
+
+
+def check_unique(items, title):
+ """Ensure all items in an iterable are identical; return that one item."""
+ its = set(items)
+ assert len(its) == 1, ("Inconsistent %s keys: %s"
+ % (title, ' '.join(map(str, sorted(its)))))
+ return its.pop()
+
+
+def fbase(fname):
+ """Strip directory and all extensions from a filename."""
+ return os.path.basename(fname).split('.', 1)[0]
+
+
+def rbase(fname):
+ """Strip directory and final extension from a filename."""
+ return os.path.basename(fname).rsplit('.', 1)[0]
diff --git a/cnvlib/coverage.py b/cnvlib/coverage.py
new file mode 100644
index 0000000..2257dca
--- /dev/null
+++ b/cnvlib/coverage.py
@@ -0,0 +1,171 @@
+"""Supporting functions for the 'antitarget' command."""
+from __future__ import absolute_import, division
+
+import logging
+import math
+import os.path
+import time
+
+from Bio._py3k import map, zip
+import pysam
+
+from .cnary import CopyNumArray as CNA
+from .rary import RegionArray as RA
+from .core import fbase
+
+from .params import NULL_LOG2_COVERAGE, READ_LEN
+
+
+def interval_coverages(bed_fname, bam_fname, by_count, min_mapq):
+ """Calculate log2 coverages in the BAM file at each interval."""
+ start_time = time.time()
+
+ # Skip processing if the BED file is empty
+ with open(bed_fname) as bed_handle:
+ for line in bed_handle:
+ if line.strip():
+ break
+ else:
+ logging.info("Skip processing %s with empty regions file %s",
+ os.path.basename(bam_fname), bed_fname)
+ return CNA.from_rows([], meta_dict={'sample_id': fbase(bam_fname)})
+
+ # Calculate average read depth in each bin
+ ic_func = (interval_coverages_count if by_count
+ else interval_coverages_pileup)
+ results = ic_func(bed_fname, bam_fname, min_mapq)
+ read_counts, cna_rows = zip(*list(results))
+
+ # Log some stats
+ tot_time = time.time() - start_time
+ tot_reads = sum(read_counts)
+ logging.info("Time: %.3f seconds (%d reads/sec, %s bins/sec)",
+ tot_time,
+ int(round(tot_reads / tot_time, 0)),
+ int(round(len(read_counts) / tot_time, 0)))
+ logging.info("Summary: #bins=%d, #reads=%d, "
+ "mean=%.4f, min=%s, max=%s ",
+ len(read_counts),
+ tot_reads,
+ (tot_reads / len(read_counts)),
+ min(read_counts),
+ max(read_counts))
+ tot_mapped_reads = bam_total_reads(bam_fname)
+ if tot_mapped_reads:
+ logging.info("Percent reads in regions: %.3f (of %d mapped)",
+ 100. * tot_reads / tot_mapped_reads,
+ tot_mapped_reads)
+ else:
+ logging.info("(Couldn't calculate total number of mapped reads)")
+
+ return CNA.from_rows(list(cna_rows),
+ meta_dict={'sample_id': fbase(bam_fname)})
+
+
+def interval_coverages_count(bed_fname, bam_fname, min_mapq):
+ """Calculate log2 coverages in the BAM file at each interval."""
+ bamfile = pysam.Samfile(bam_fname, 'rb')
+ for chrom, subregions in RA.read(bed_fname).by_chromosome():
+ logging.info("Processing chromosome %s of %s",
+ chrom, os.path.basename(bam_fname))
+ for _chrom, start, end, name, in subregions.coords(["name"]):
+ count, depth = region_depth_count(bamfile, chrom, start, end,
+ min_mapq)
+ yield [count,
+ (chrom, start, end, name,
+ math.log(depth, 2) if depth else NULL_LOG2_COVERAGE)]
+
+
+def region_depth_count(bamfile, chrom, start, end, min_mapq):
+ """Calculate depth of a region via pysam count.
+
+ i.e. counting the number of read starts in a region, then scaling for read
+ length and region width to estimate depth.
+
+ Coordinates are 0-based, per pysam.
+ """
+ def filter_read(read):
+ """True if the given read should be counted towards coverage."""
+ return not (read.is_duplicate
+ or read.is_secondary
+ or read.is_unmapped
+ or read.is_qcfail
+ or read.mapq < min_mapq)
+
+ # Count the number of read midpoints in the interval
+ count = sum((start <= (read.pos + .5*read.rlen) <= end and filter_read(read))
+ for read in bamfile.fetch(reference=chrom,
+ start=start, end=end))
+ # Scale read counts to region length
+ # Depth := #Bases / Span
+ depth = (READ_LEN * count / (end - start)
+ if end > start else 0)
+ return count, depth
+
+
+def interval_coverages_pileup(bed_fname, bam_fname, min_mapq):
+ """Calculate log2 coverages in the BAM file at each interval."""
+ logging.info("Processing reads in %s", os.path.basename(bam_fname))
+ for chrom, start, end, name, count, depth in bedcov(bed_fname, bam_fname,
+ min_mapq):
+ yield [count,
+ (chrom, start, end, name,
+ math.log(depth, 2) if depth else NULL_LOG2_COVERAGE)]
+
+
+def bedcov(bed_fname, bam_fname, min_mapq):
+ """Calculate depth of all regions in a BED file via samtools (pysam) bedcov.
+
+ i.e. mean pileup depth across each region.
+ """
+ # Count bases in each region; exclude low-MAPQ reads
+ if min_mapq > 0:
+ bedcov_args = ['-Q', str(min_mapq)]
+ else:
+ bedcov_args = []
+ try:
+ lines = pysam.bedcov(bed_fname, bam_fname, *bedcov_args)
+ except pysam.SamtoolsError as exc:
+ raise ValueError("Failed processing %r coverages in %r regions. PySAM error: %s"
+ % (bam_fname, bed_fname, exc))
+ if not lines:
+ raise ValueError("BED file %r sequence IDs don't match any in BAM file %r"
+ % (bed_fname, bam_fname))
+ # Return an iterable...
+ if isinstance(lines, basestring):
+ lines = lines.splitlines()
+ for line in lines:
+ fields = line.split('\t')
+ if len(fields) == 5:
+ chrom, start_s, end_s, name, basecount_s = fields
+ elif len(fields) == 4:
+ chrom, start_s, end_s, basecount_s = fields
+ name = "-"
+ else:
+ raise RuntimeError("Bad line from bedcov:\n" + line)
+ start, end, basecount = map(int, (start_s, end_s, basecount_s.strip()))
+ span = end - start
+ if span > 0:
+ # Algebra from above
+ count = basecount / READ_LEN
+ mean_depth = basecount / span
+ else:
+ # User-supplied bins might be oddly constructed
+ count = mean_depth = 0
+ yield chrom, start, end, name, count, mean_depth
+
+
+def bam_total_reads(bam_fname):
+ """Count the total number of mapped reads in a BAM file.
+
+ Uses the BAM index to do this quickly.
+ """
+ lines = pysam.idxstats(bam_fname)
+ if isinstance(lines, basestring):
+ lines = lines.splitlines()
+ tot_mapped_reads = 0
+ for line in lines:
+ _seqname, _seqlen, nmapped, _nunmapped = line.split()
+ tot_mapped_reads += int(nmapped)
+ return tot_mapped_reads
+
diff --git a/cnvlib/diagram.py b/cnvlib/diagram.py
new file mode 100644
index 0000000..c19172f
--- /dev/null
+++ b/cnvlib/diagram.py
@@ -0,0 +1,226 @@
+"""Chromosome diagram drawing functions.
+
+This uses and abuses Biopython's BasicChromosome module. It depends on
+ReportLab, too, so we isolate this functionality here so that the rest of CNVkit
+will run without it. (And also to keep the codebase tidy.)
+"""
+from __future__ import division
+import collections
+import math
+import warnings
+
+from Bio.Graphics import BasicChromosome as BC
+from reportlab.graphics import renderPDF
+from reportlab.lib import colors
+from reportlab.lib.units import inch
+from reportlab.pdfgen import canvas
+
+from . import params, plots, reports
+
+# Silence Biopython's whinging
+from Bio import BiopythonWarning
+warnings.simplefilter('ignore', BiopythonWarning)
+
+TELOMERE_LENGTH = 6e6 # For illustration only
+CHROM_FATNESS = 0.3
+PAGE_SIZE = (11.0*inch, 8.5*inch)
+
+
+def create_diagram(cnarr, segarr, threshold, min_probes, outfname, male_reference):
+ """Create the diagram."""
+ if cnarr and segarr:
+ do_both = True # Draw segments on left, probes on right.
+ cnarr_is_seg = False # Are probes actually the segmented values?
+ else:
+ if cnarr:
+ cnarr_is_seg = False
+ elif segarr:
+ cnarr = segarr
+ cnarr_is_seg = True
+ else:
+ raise ValueError("Must specify a filename as an argument or with "
+ "the '-s' option, or both. You did neither.")
+ do_both = False
+
+ # Label genes where copy ratio value exceeds threshold
+ cnarr = cnarr.shift_xx(male_reference)
+ if cnarr_is_seg:
+ sel = cnarr.data[(cnarr.data.log2.abs() >= threshold) &
+ ~cnarr.data.gene.isin(params.IGNORE_GENE_NAMES)]
+ gainloss = [(s.gene, s.chromosome, s.start, s.end, s.log2, s.probes)
+ for s in sel.itertuples(index=False)]
+ elif segarr:
+ segarr = segarr.shift_xx(male_reference)
+ gainloss = reports.gainloss_by_segment(cnarr, segarr, threshold)
+ else:
+ gainloss = reports.gainloss_by_gene(cnarr, threshold)
+ gene_labels = [gl_row[0] for gl_row in gainloss if gl_row[5] >= min_probes]
+ # NB: If multiple segments cover the same gene (gene contains breakpoints),
+ # all those segments are marked as "hits". We'll uniquify them.
+ # TODO - use different logic to only label the gene's signficant segment(s)
+ seen_genes = set()
+
+ # Consolidate genes & coverage values as chromosome features
+ features = collections.defaultdict(list)
+ strand = 1 if do_both else None # Draw on the chr. right half or full width
+ chrom_sizes = plots.chromosome_sizes(cnarr)
+ if not cnarr_is_seg:
+ cnarr = cnarr.squash_genes()
+ for row in cnarr:
+ if row.start - 1 >= 0 and row.end <= chrom_sizes[row.chromosome]: # Sanity check
+ if row.gene in gene_labels and row.gene not in seen_genes:
+ seen_genes.add(row.gene)
+ feat_name = row.gene
+ if "," in feat_name:
+ # TODO - line-wrap multi-gene labels (reportlab won't do \n)
+ feat_name = feat_name.replace(",", ", ")
+ else:
+ feat_name = None
+ features[row.chromosome].append(
+ (row.start - 1, row.end, strand, feat_name,
+ colors.Color(*plots.cvg2rgb(row.log2, not cnarr_is_seg))))
+ if do_both:
+ # Draw segments in the left half of each chromosome (strand -1)
+ for chrom, segrows in segarr.by_chromosome():
+ for srow in segrows:
+ if srow.start - 1 >= 0 and srow.end <= chrom_sizes[chrom]: # Sanity check
+ features[chrom].append(
+ (srow.start - 1, srow.end, -1, None,
+ colors.Color(*plots.cvg2rgb(srow.log2, False))))
+
+ # Generate the diagram PDF
+ if not outfname:
+ outfname = cnarr.sample_id + '-diagram.pdf'
+ drawing = build_chrom_diagram(features, chrom_sizes, cnarr.sample_id)
+ cvs = canvas.Canvas(outfname, pagesize=PAGE_SIZE)
+ renderPDF.draw(drawing, cvs, 0, 0)
+ cvs.showPage()
+ cvs.save()
+ return outfname
+
+
+def build_chrom_diagram(features, chr_sizes, sample_id):
+ """Create a PDF of color-coded features on chromosomes."""
+ max_chr_len = max(chr_sizes.values())
+
+ chr_diagram = BC.Organism()
+ chr_diagram.page_size = PAGE_SIZE
+ chr_diagram.title_size = 18
+
+ for chrom, length in chr_sizes.items():
+ chrom_features = features.get(chrom)
+ if not chrom_features:
+ continue
+ body = BC.AnnotatedChromosomeSegment(length, chrom_features)
+ body.label_size = 4
+ body.scale = length
+ body.chr_percent = CHROM_FATNESS
+
+ # Create opening and closing telomeres
+ tel_start = BC.TelomereSegment()
+ tel_start.scale = TELOMERE_LENGTH
+ tel_start.chr_percent = CHROM_FATNESS
+ tel_end = BC.TelomereSegment(inverted=True)
+ tel_end.scale = TELOMERE_LENGTH
+ tel_end.chr_percent = CHROM_FATNESS
+
+ # Assemble the chromosome diagram in order
+ cur_chromosome = BC.Chromosome(chrom)
+ cur_chromosome.title_size = 14
+ # Set the scale to the MAXIMUM length plus the two telomeres in bp,
+ # want the same scale used on all chromosomes so they can be
+ # compared to each other
+ cur_chromosome.scale_num = max_chr_len + 2 * TELOMERE_LENGTH
+ cur_chromosome.add(tel_start)
+ cur_chromosome.add(body)
+ cur_chromosome.add(tel_end)
+ chr_diagram.add(cur_chromosome)
+
+ title = "Sample " + sample_id
+ return bc_organism_draw(chr_diagram, title)
+
+
+def bc_organism_draw(org, title, wrap=12):
+ """Modified copy of Bio.Graphics.BasicChromosome.Organism.draw.
+
+ Instead of stacking chromosomes horizontally (along the x-axis), stack rows
+ vertically, then proceed with the chromosomes within each row.
+
+ Arguments:
+
+ - title: The output title of the produced document.
+ """
+ margin_top = 1.25*inch
+ margin_bottom = 0.1*inch
+ margin_side = 0.5*inch
+
+ width, height = org.page_size
+ cur_drawing = BC.Drawing(width, height)
+
+ # Draw the title text
+ title_string = BC.String(width / 2, height - margin_top + .5*inch, title)
+ title_string.fontName = 'Helvetica-Bold'
+ title_string.fontSize = org.title_size
+ title_string.textAnchor = "middle"
+ cur_drawing.add(title_string)
+
+ # Layout subcomponents (individual chromosomes), wrapping into rows
+ if len(org._sub_components) > 0:
+ nrows = math.ceil(len(org._sub_components) / wrap)
+ x_pos_change = (width - 2 * margin_side) / wrap
+ y_pos_change = (height - margin_top - margin_bottom) / nrows
+
+ cur_x_pos = margin_side
+ cur_row = 0
+ for i, sub_component in enumerate(org._sub_components):
+ if i % wrap == 0 and i != 0:
+ cur_row += 1
+ cur_x_pos = margin_side
+ # Set the page coordinates of the chromosome drawing
+ sub_component.start_x_position = cur_x_pos + 0.05 * x_pos_change
+ sub_component.end_x_position = cur_x_pos + 0.95 * x_pos_change
+ sub_component.start_y_position = (height - margin_top
+ - y_pos_change * cur_row)
+ sub_component.end_y_position = (margin_bottom
+ + y_pos_change * (nrows - cur_row - 1))
+ # Render the chromosome drawing
+ sub_component.draw(cur_drawing)
+ # Update the locations for the next chromosome
+ cur_x_pos += x_pos_change
+
+ # Draw a legend
+ # (Rect coordinates are: left, bottom, width, height)
+ # Bounding box -- near-bottom, center
+ cur_drawing.add(BC.Rect(width/2 - .8*inch, .5*inch, 1.6*inch, .4*inch,
+ fillColor=colors.white))
+ # Red box & label -- in left half of bounding box
+ cur_drawing.add(BC.Rect(width/2 - .7*inch, .6*inch, .2*inch, .2*inch,
+ fillColor=colors.Color(.8, .2, .2)))
+ cur_drawing.add(BC.String(width/2 - .42*inch, .65*inch, "Gain",
+ fontName='Helvetica', fontSize=12))
+ # Blue box & label -- in right half of bounding box
+ cur_drawing.add(BC.Rect(width/2 + .07*inch, .6*inch, .2*inch, .2*inch,
+ fillColor=colors.Color(.2, .2, .8)))
+ cur_drawing.add(BC.String(width/2 + .35*inch, .65*inch, "Loss",
+ fontName='Helvetica', fontSize=12))
+
+ # Let the caller take care of writing to the file...
+ return cur_drawing
+
+
+def bc_chromosome_draw_label(self, cur_drawing, label_name):
+ """Monkeypatch to Bio.Graphics.BasicChromosome.Chromosome._draw_label.
+
+ Draw a label for the chromosome. Mod: above the chromosome, not below.
+ """
+ # Center on chromosome image
+ x_position = 0.5 * (self.start_x_position + self.end_x_position)
+ # Place at the bottom of the diagram?
+ y_position = self.start_y_position + 0.1*inch # was: self.end_y_position
+ label_string = BC.String(x_position, y_position, label_name)
+ label_string.fontName = 'Times-BoldItalic'
+ label_string.fontSize = self.title_size
+ label_string.textAnchor = 'middle'
+ cur_drawing.add(label_string)
+
+BC.Chromosome._draw_label = bc_chromosome_draw_label
diff --git a/cnvlib/export.py b/cnvlib/export.py
new file mode 100644
index 0000000..e96218a
--- /dev/null
+++ b/cnvlib/export.py
@@ -0,0 +1,499 @@
+"""Export CNVkit objects and files to other formats."""
+from __future__ import absolute_import, division, print_function
+
+import collections
+import logging
+import time
+
+import numpy as np
+import pandas as pd
+from Bio._py3k import map, range, zip
+
+from . import call, core, params
+from .cnary import CopyNumArray as CNA
+from .vary import VariantArray as VA
+from ._version import __version__
+
+
+def merge_samples(filenames):
+ """Merge probe values from multiple samples into a 2D table (of sorts).
+
+ Input:
+ dict of {sample ID: (probes, values)}
+ Output:
+ list-of-tuples: (probe, log2 coverages...)
+ """
+ def label_with_gene(cnarr):
+ row2label = lambda row: "{}:{}-{}:{}".format(
+ row.chromosome, row.start, row.end, row.gene)
+ return cnarr.data.apply(row2label, axis=1)
+
+ if not filenames:
+ return []
+ first_cnarr = CNA.read(filenames[0])
+ out_table = first_cnarr.data.loc[:, ["chromosome", "start", "end", "gene"]]
+ out_table["label"] = label_with_gene(first_cnarr)
+ out_table[first_cnarr.sample_id] = first_cnarr["log2"]
+ for fname in filenames[1:]:
+ cnarr = CNA.read(fname)
+ # Verify labels match
+ labels = label_with_gene(cnarr)
+ if not (labels == out_table["label"]).all():
+ raise ValueError("Mismatched row coordinates in %s" % fname)
+ # Copy the next column by sample ID
+ if cnarr.sample_id in out_table.columns:
+ raise ValueError("Duplicate sample ID: %s" % cnarr.sample_id)
+ out_table[cnarr.sample_id] = cnarr["log2"]
+ del cnarr
+ return out_table
+
+
+# Supported formats:
+
+def fmt_cdt(sample_ids, table):
+ """Format as CDT."""
+ outheader = ['GID', 'CLID', 'NAME', 'GWEIGHT'] + sample_ids
+ header2 = ['AID', '', '', '']
+ header2.extend(['ARRY' + str(i).zfill(3) + 'X'
+ for i in range(len(sample_ids))])
+ outrows = [header2]
+ outtable = pd.concat([
+ pd.DataFrame({
+ "GID": table.index.apply(lambda x: "GENE%dX" % x),
+ "CLID": table.index.apply(lambda x: "IMAGE:%d" % x),
+ "NAME": table["label"],
+ "GWEIGHT": 1,
+ }),
+ table.drop(["chromosome", "start", "end", "gene", "label"],
+ axis=1)],
+ axis=1)
+ outrows.extend(outtable.itertuples(index=False))
+ return outheader, outrows
+
+
+# TODO
+def fmt_gct(sample_ids, table):
+ return NotImplemented
+
+
+def fmt_jtv(sample_ids, table):
+ """Format for Java TreeView."""
+ outheader = ["CloneID", "Name"] + sample_ids
+ outtable = pd.concat([
+ pd.DataFrame({
+ "CloneID": "IMAGE:",
+ "Name": table["label"],
+ }),
+ table.drop(["chromosome", "start", "end", "gene", "label"],
+ axis=1)],
+ axis=1)
+ outrows = outtable.itertuples(index=False)
+ return outheader, outrows
+
+
+# Special cases
+
+def export_nexus_basic(sample_fname):
+ """Biodiscovery Nexus Copy Number "basic" format.
+
+ Only represents one sample per file.
+ """
+ cnarr = CNA.read(sample_fname)
+ out_table = cnarr.data.loc[:, ['chromosome', 'start', 'end', 'gene', 'log2']]
+ out_table['probe'] = cnarr.labels()
+ return out_table
+
+
+def export_nexus_ogt(sample_fname, vcf_fname, sample_id,
+ min_depth=20, min_weight=0.0):
+ """Biodiscovery Nexus Copy Number "Custom-OGT" format.
+
+ To create the b-allele frequencies column, alterate allele frequencies from
+ the VCF are aligned to the .cnr file bins. Bins that contain no variants
+ are left blank; if a bin contains multiple variants, then the frequencies
+ are all "mirrored" to be above .5, then the median of those values is taken.
+ """
+ cnarr = CNA.read(sample_fname)
+ if min_weight and "weight" in cnarr:
+ mask_low_weight = (cnarr["weight"] < min_weight)
+ logging.info("Dropping %d bins with weight below %f",
+ mask_low_weight.sum(), min_weight)
+ cnarr.data = cnarr.data[~mask_low_weight]
+ varr = VA.read_vcf(vcf_fname, sample_id=sample_id or cnarr.sample_id,
+ min_depth=min_depth, skip_hom=True, skip_somatic=True)
+ bafs = cnarr.match_to_bins(varr, 'alt_freq', np.nan,
+ summary_func=mirrored_baf_median)
+ logging.info("Placed %d variants into %d bins",
+ sum(~np.isnan(bafs)), len(cnarr))
+ out_table = cnarr.data.loc[:, ['chromosome', 'start', 'end', 'log2']]
+ out_table = out_table.rename(columns={
+ "chromosome": "Chromosome",
+ "start": "Position",
+ "end": "Position",
+ "log2": "Log R Ratio",
+ })
+ out_table["B-Allele Frequency"] = bafs
+ return out_table
+
+
+def mirrored_baf_median(vals):
+ shift = np.median(np.abs(vals - .5))
+ if np.median(vals) > .5:
+ return .5 + shift
+ else:
+ return .5 - shift
+
+
+def export_seg(sample_fnames):
+ """SEG format for copy number segments.
+
+ Segment breakpoints are not the same across samples, so samples are listed
+ in serial with the sample ID as the left column.
+ """
+ out_tables = []
+ chrom_ids = None
+ for fname in sample_fnames:
+ segments = CNA.read(fname)
+ if chrom_ids is None:
+ # Create & store
+ chrom_ids = create_chrom_ids(segments)
+ else:
+ # Verify
+ core.assert_equal("Segment chromosome names differ",
+ previous=chrom_ids.keys(),
+ current=create_chrom_ids(segments).keys())
+ table = segments.data.loc[:, ["start", "end"]]
+ table["ID"] = segments.sample_id
+ table["mean"] = segments.data["log2"]
+ table["chromosome"] = [chrom_ids[chrom]
+ for chrom in segments["chromosome"]]
+ if "probes" in segments:
+ table["num_probes"] = segments["probes"]
+ sorted_cols = ["ID", "chromosome", "start", "end", "num_probes",
+ "mean"]
+ else:
+ sorted_cols = ["ID", "chromosome", "start", "end", "mean"]
+ out_tables.append(table.reindex(columns=sorted_cols))
+ return pd.concat(out_tables)
+
+
+def create_chrom_ids(segments):
+ """Map chromosome names to integers in the order encountered."""
+ mapping = collections.OrderedDict()
+ curr_idx = 1
+ for chrom in segments.chromosome:
+ if chrom not in mapping:
+ mapping[chrom] = curr_idx
+ curr_idx += 1
+ return mapping
+
+
+# _____________________________________________________________________________
+# BED
+
+def export_bed(segments, ploidy, is_reference_male, is_sample_female,
+ label, show):
+ """Convert a copy number array to a BED-like DataFrame.
+
+ For each region in each sample (possibly filtered according to `show`),
+ the columns are:
+
+ - reference sequence name
+ - start (0-indexed)
+ - end
+ - sample name or given label
+ - integer copy number
+
+ By default (show="ploidy"), skip regions where copy number is the default
+ ploidy, i.e. equal to 2 or the value set by --ploidy.
+ If show="variant", skip regions where copy number is neutral, i.e. equal to
+ the reference ploidy on autosomes, or half that on sex chromosomes.
+ """
+ out = segments.data.loc[:, ["chromosome", "start", "end"]]
+ out["label"] = label
+ out["ncopies"] = (segments["cn"] if "cn" in segments
+ else np.rint(call.absolute_pure(segments, ploidy,
+ is_reference_male)))
+ if show == "ploidy":
+ # Skip regions of default ploidy
+ out = out[out["ncopies"] != ploidy]
+ elif show == "variant":
+ # Skip regions of non-neutral copy number
+ exp_copies = call.absolute_expect(segments, ploidy, is_sample_female)
+ out = out[out["ncopies"] != exp_copies]
+ return out
+
+
+# _____________________________________________________________________________
+# VCF
+
+VCF_HEADER = """\
+##fileformat=VCFv4.0
+##fileDate={date}
+##source=CNVkit v{version}
+##INFO=<ID=CIEND,Number=2,Type=Integer,Description="Confidence interval around END for imprecise variants">
+##INFO=<ID=CIPOS,Number=2,Type=Integer,Description="Confidence interval around POS for imprecise variants">
+##INFO=<ID=END,Number=1,Type=Integer,Description="End position of the variant described in this record">
+##INFO=<ID=IMPRECISE,Number=0,Type=Flag,Description="Imprecise structural variation">
+##INFO=<ID=SVLEN,Number=1,Type=Integer,Description="Difference in length between REF and ALT alleles">
+##INFO=<ID=SVTYPE,Number=1,Type=String,Description="Type of structural variant">
+##INFO=<ID=FOLD_CHANGE,Number=1,Type=Float,Description="Fold change">
+##INFO=<ID=FOLD_CHANGE_LOG,Number=1,Type=Float,Description="Log fold change">
+##INFO=<ID=PROBES,Number=1,Type=Integer,Description="Number of probes in CNV">
+##ALT=<ID=DEL,Description="Deletion">
+##ALT=<ID=DUP,Description="Duplication">
+##ALT=<ID=CNV,Description="Copy number variable region">
+##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
+##FORMAT=<ID=GQ,Number=1,Type=Float,Description="Genotype quality">
+##FORMAT=<ID=CN,Number=1,Type=Integer,Description="Copy number genotype for imprecise events">
+##FORMAT=<ID=CNQ,Number=1,Type=Float,Description="Copy number genotype quality for imprecise events">
+""".format(date=time.strftime("%Y%m%d"), version=__version__)
+# #CHROM POS ID REF ALT QUAL FILTER INFO FORMAT NA00001
+# 1 2827693 . CCGTGGATGCGGGGACCCGCATCCCCTCTCCCTTCACAGCTGAGTGACCCACATCCCCTCTCCCCTCGCA C . PASS SVTYPE=DEL;END=2827680;BKPTID=Pindel_LCS_D1099159;HOMLEN=1;HOMSEQ=C;SVLEN=-66 GT:GQ 1/1:13.9
+# 2 321682 . T <DEL> 6 PASS IMPRECISE;SVTYPE=DEL;END=321887;SVLEN=-105;CIPOS=-56,20;CIEND=-10,62 GT:GQ 0/1:12
+# 3 12665100 . A <DUP> 14 PASS IMPRECISE;SVTYPE=DUP;END=12686200;SVLEN=21100;CIPOS=-500,500;CIEND=-500,500 GT:GQ:CN:CNQ ./.:0:3:16.2
+# 4 18665128 . T <DUP:TANDEM> 11 PASS IMPRECISE;SVTYPE=DUP;END=18665204;SVLEN=76;CIPOS=-10,10;CIEND=-10,10 GT:GQ:CN:CNQ ./.:0:5:8.3
+
+
+def export_vcf(segments, ploidy, is_reference_male, is_sample_female,
+ sample_id=None):
+ """Convert segments to Variant Call Format.
+
+ For now, only 1 sample per VCF. (Overlapping CNVs seem tricky.)
+
+ Spec: https://samtools.github.io/hts-specs/VCFv4.2.pdf
+ """
+ vcf_columns = ["#CHROM", "POS", "ID", "REF", "ALT", "QUAL", "FILTER",
+ "INFO", "FORMAT", sample_id or segments.sample_id]
+ vcf_rows = segments2vcf(segments, ploidy, is_reference_male,
+ is_sample_female)
+ table = pd.DataFrame.from_records(vcf_rows, columns=vcf_columns)
+ vcf_body = table.to_csv(sep='\t', header=True, index=False,
+ float_format="%.3g")
+ return VCF_HEADER, vcf_body
+
+
+def segments2vcf(segments, ploidy, is_reference_male, is_sample_female):
+ """Convert copy number segments to VCF records."""
+ out_dframe = segments.data.loc[:, ["chromosome", "end", "log2", "probes"]]
+ if "cn" in segments:
+ out_dframe["ncopies"] = segments["cn"]
+ abs_expect = call.absolute_expect(segments, ploidy, is_sample_female)
+
+ else:
+ abs_dframe = call.absolute_dataframe(segments, ploidy, 1.0,
+ is_reference_male,
+ is_sample_female)
+ out_dframe["ncopies"] = np.rint(abs_dframe["absolute"])
+ abs_expect = abs_dframe["expect"]
+ idx_losses = (out_dframe["ncopies"] < abs_expect)
+
+ starts = segments.start.copy()
+ starts[starts == 0] = 1
+ out_dframe["start"] = starts
+
+ svlen = segments.end - segments.start
+ svlen[idx_losses] *= -1
+ out_dframe["svlen"] = svlen
+
+ out_dframe["svtype"] = "DUP"
+ out_dframe.loc[idx_losses, "svtype"] = "DEL"
+
+ out_dframe["format"] = "GT:GQ:CN:CNQ"
+ out_dframe.loc[idx_losses, "format"] = "GT:GQ" # :CN:CNQ ?
+
+ # Reformat this data to create INFO and genotype
+ # TODO be more clever about this
+ for out_row, abs_exp in zip(out_dframe.itertuples(index=False), abs_expect):
+ if (out_row.ncopies == abs_exp or
+ # Survive files from buggy v0.7.1 (#53)
+ not str(out_row.probes).isdigit()):
+ # Skip regions of neutral copy number
+ continue # or "CNV" for subclonal?
+
+ if out_row.ncopies > abs_exp:
+ genotype = "0/1:0:%d:%d" % (out_row.ncopies, out_row.probes)
+ elif out_row.ncopies < abs_exp:
+ # TODO XXX handle non-diploid ploidies, haploid chroms
+ if out_row.ncopies == 0:
+ # Complete deletion, 0 copies
+ gt = "1/1"
+ else:
+ # Single copy deletion
+ gt = "0/1"
+ genotype = "%s:%d" % (gt, out_row.probes)
+
+ info = ";".join(["IMPRECISE",
+ "SVTYPE=%s" % out_row.svtype,
+ "END=%d" % out_row.end,
+ "SVLEN=%d" % out_row.svlen,
+ "FOLD_CHANGE=%f" % 2.0 ** out_row.log2,
+ "FOLD_CHANGE_LOG=%f" % out_row.log2,
+ "PROBES=%d" % out_row.probes
+ # CIPOS=-56,20;CIEND=-10,62
+ ])
+
+ yield (out_row.chromosome, out_row.start, '.', 'N',
+ "<%s>" % out_row.svtype, '.', '.',
+ info, out_row.format, genotype)
+
+
+# _____________________________________________________________________________
+# THetA
+
+def export_theta(tumor_segs, normal_cn):
+ """Convert tumor segments and normal .cnr or reference .cnn to THetA input.
+
+ Follows the THetA segmentation import script but avoid repeating the
+ pileups, since we already have the mean depth of coverage in each target
+ bin.
+
+ The options for average depth of coverage and read length do not matter
+ crucially for proper operation of THetA; increased read counts per bin
+ simply increase the confidence of THetA's results.
+
+ THetA2 input format is tabular, with columns:
+ ID, chrm, start, end, tumorCount, normalCount
+
+ where chromosome IDs ("chrm") are integers 1 through 24.
+ """
+ # Drop any chromosomes that are not integer or XY
+ # THetA hard-codes X & Y, so we can, too
+ xy_names = (["chrX", "chrY"]
+ if tumor_segs.chromosome.iat[0].startswith('chr')
+ else ["X", "Y"])
+ tumor_segs = tumor_segs.autosomes(also=xy_names)
+ if normal_cn:
+ normal_cn = normal_cn.autosomes(also=xy_names)
+
+ table = tumor_segs.data.loc[:, ["start", "end"]]
+
+ # Convert chromosome names to 1-based integer indices
+ chr2idx = {c: i+1
+ for i, c in enumerate(tumor_segs.chromosome.drop_duplicates())}
+ table["chrm"] = tumor_segs.chromosome.replace(chr2idx)
+ # Unique string identifier for each row, e.g. "start_1_93709:end_1_19208166"
+ table["#ID"] = ["start_%d_%d:end_%d_%d"
+ % (row.chrm, row.start, row.chrm, row.end)
+ for row in table.itertuples(index=False)]
+
+ # Calculate/estimate per-segment read counts in tumor and normal samples
+ ref_means, nbins = ref_means_nbins(tumor_segs, normal_cn)
+ table["tumorCount"] = theta_read_counts(tumor_segs.log2, nbins)
+ table["normalCount"] = theta_read_counts(ref_means, nbins)
+ return table[["#ID", "chrm", "start", "end", "tumorCount", "normalCount"]]
+
+
+def ref_means_nbins(tumor_segs, normal_cn):
+ """Extract segments' reference mean log2 values and probe counts.
+
+ Code paths::
+
+ wt_mdn wt_old probes norm -> norm, nbins
+ + * * - 0, wt_mdn
+ - + + - 0, wt_old * probes
+ - + - - 0, wt_old * size?
+ - - + - 0, probes
+ - - - - 0, size?
+
+ + - + + norm, probes
+ + - - + norm, bin counts
+ - + + + norm, probes
+ - + - + norm, bin counts
+ - - + + norm, probes
+ - - - + norm, bin counts
+ """
+ if normal_cn:
+ subarrs = [subarr for _seg, subarr in normal_cn.by_ranges(tumor_segs)]
+ # For the normal/reference bin count, take the mean of the bin values
+ # within each segment so that segments match between tumor and normal.
+ # ENH: weighted mean, like gainloss
+ ref_means = np.asarray([s.log2.mean() for s in subarrs])
+ if "probes" in tumor_segs:
+ nbins = tumor_segs["probes"]
+ else:
+ nbins = np.asarray([len(s) for s in subarrs])
+ else:
+ # Assume neutral reference log2 across all segments
+ # (weights already account for reference log2)
+ ref_means = np.zeros(len(tumor_segs))
+ if "weight" in tumor_segs and (tumor_segs["weight"] > 1.0).any():
+ # Segment weights are already multiplied by probe counts
+ nbins = tumor_segs["weight"]
+ # Rescale to average 1.0 (we'll do the same below, too)
+ nbins /= nbins.max() / nbins.mean()
+ else:
+ if "probes" in tumor_segs:
+ nbins = tumor_segs["probes"]
+ else:
+ logging.warn("No probe counts in tumor segments file and no "
+ "normal reference given; guessing normal "
+ "read-counts-per-segment from segment sizes")
+ sizes = tumor_segs.end - tumor_segs.start
+ nbins = sizes / sizes.mean()
+ if "weight" in tumor_segs:
+ # Already checked -- these are old-style weights that were not
+ # multiplied by `probes` originally
+ nbins *= tumor_segs["weight"] / tumor_segs["weight"].mean()
+ return ref_means, nbins
+
+
+def theta_read_counts(log2_ratio, nbins,
+ # These two scaling factors don't meaningfully affect
+ # THetA's calculation unless they're too small
+ avg_depth=500, avg_bin_width=200):
+ """Calculate segments' read counts from log2-ratios.
+
+ Math:
+ nbases = read_length * read_count
+ and
+ nbases = bin_width * read_depth
+ where
+ read_depth = read_depth_ratio * avg_depth
+
+ So:
+ read_length * read_count = bin_width * read_depth
+ read_count = bin_width * read_depth / read_length
+ """
+ read_depth = (2 ** log2_ratio) * avg_depth
+ read_count = nbins * avg_bin_width * read_depth / params.READ_LEN
+ return read_count.round().astype('int')
+
+
+def export_theta_snps(varr):
+ """Generate THetA's SNP per-allele read count "formatted.txt" files."""
+ # Drop any chromosomes that are not integer or XY
+ varr = varr.autosomes(also=(["chrX", "chrY"]
+ if varr.chromosome.iat[0].startswith("chr")
+ else ["X", "Y"]))
+ # Skip indels
+ varr = varr[(varr["ref"].str.len() == 1) & (varr["alt"].str.len() == 1)]
+ # Drop rows with any NaN
+ varr.data.dropna(subset=["depth", "n_depth", "alt_count", "n_alt_count"],
+ inplace=True)
+ # Avoid weird situation I've seen on alt contigs
+ varr = varr[varr["depth"] >= varr["alt_count"]]
+ # Reformat for THetA2
+ for depth_key, alt_key in (("depth", "alt_count"),
+ ("n_depth", "n_alt_count")):
+ # Extract the SNP allele counts that THetA2 uses
+ table = varr.data.loc[:, ("chromosome", "start", depth_key, alt_key)]
+ table["ref_depth"] = (table[depth_key] - table[alt_key]).astype("int")
+ table[alt_key] = table[alt_key].astype("int")
+ table = table.loc[:, ("chromosome", "start", "ref_depth", alt_key)]
+ table.columns = ["#Chrm", "Pos", "Ref_Allele", "Mut_Allele"]
+ yield table
+
+
+# _____________________________________________________________________________
+
+EXPORT_FORMATS = {
+ 'cdt': fmt_cdt,
+ # 'gct': fmt_gct,
+ 'jtv': fmt_jtv,
+ 'nexus-basic': export_nexus_basic,
+ 'nexus-ogt': export_nexus_ogt,
+ 'seg': export_seg,
+ 'theta': export_theta,
+ 'vcf': export_vcf,
+}
diff --git a/cnvlib/fix.py b/cnvlib/fix.py
new file mode 100644
index 0000000..1139022
--- /dev/null
+++ b/cnvlib/fix.py
@@ -0,0 +1,233 @@
+"""Supporting functions for the 'fix' command."""
+from __future__ import absolute_import, division, print_function
+
+import logging
+
+import numpy as np
+import pandas as pd
+
+from . import params, smoothing
+
+
+def load_adjust_coverages(pset, ref_pset, skip_low,
+ fix_gc, fix_edge, fix_rmask):
+ """Load and filter probe coverages; correct using reference and GC."""
+ if 'gc' in pset:
+ # Don't choke on Picard-derived files that have the GC column
+ pset = pset.drop_extra_columns()
+
+ # No corrections needed if there are no data rows (e.g. no antitargets)
+ if not len(pset):
+ return pset
+
+ ref_matched = match_ref_to_probes(ref_pset, pset)
+
+ # Drop probes that had poor coverage in the pooled reference
+ ok_cvg_indices = ~mask_bad_probes(ref_matched)
+ logging.info("Keeping %d of %d bins", sum(ok_cvg_indices), len(ref_matched))
+ pset = pset[ok_cvg_indices]
+ ref_matched = ref_matched[ok_cvg_indices]
+
+ # Apply corrections for known systematic biases in coverage
+ pset.center_all(skip_low=skip_low)
+ # Skip bias corrections if most bins have no coverage (e.g. user error)
+ if (pset['log2'] > params.NULL_LOG2_COVERAGE - params.MIN_REF_COVERAGE
+ ).sum() <= len(pset) // 2:
+ logging.warn("WARNING: most bins have no or very low coverage; "
+ "check that the right BED file was used")
+ else:
+ if fix_gc:
+ if 'gc' in ref_matched:
+ logging.info("Correcting for GC bias...")
+ pset = center_by_window(pset, .1, ref_matched['gc'])
+ else:
+ logging.warn("WARNING: Skipping correction for GC bias")
+ if fix_edge:
+ logging.info("Correcting for density bias...")
+ edge_bias = get_edge_bias(pset, params.INSERT_SIZE)
+ pset = center_by_window(pset, .1, edge_bias)
+ if fix_rmask:
+ if 'rmask' in ref_matched:
+ logging.info("Correcting for RepeatMasker bias...")
+ pset = center_by_window(pset, .1, ref_matched['rmask'])
+ else:
+ logging.warn("WARNING: Skipping correction for RepeatMasker bias")
+
+ # Normalize coverages according to the reference
+ # (Subtract the reference log2 copy number to get the log2 ratio)
+ pset.data['log2'] -= ref_matched['log2']
+ pset.center_all(skip_low=skip_low)
+ return apply_weights(pset, ref_matched)
+
+
+def mask_bad_probes(probes):
+ """Flag the probes with excessively low or inconsistent coverage.
+
+ Returns a bool array where True indicates probes that failed the checks.
+ """
+ mask = ((probes['log2'] < params.MIN_REF_COVERAGE) |
+ (probes['log2'] > -params.MIN_REF_COVERAGE) |
+ (probes['spread'] > params.MAX_REF_SPREAD))
+ return mask
+
+
+def match_ref_to_probes(ref_pset, probes):
+ """Filter the reference probes to match the target or antitarget probe set.
+ """
+ probes_labeled = probes.data.set_index(pd.Index(probes.coords()))
+ ref_labeled = ref_pset.data.set_index(pd.Index(ref_pset.coords()))
+ # Safety
+ for dset, name in ((probes_labeled, "probe"),
+ (ref_labeled, "reference")):
+ dupes = dset.index.duplicated()
+ if dupes.any():
+ raise ValueError("Duplicated genomic coordinates in " + name +
+ " set:\n" + "\n".join(map(str, dset.index[dupes])))
+ ref_matched = ref_labeled.reindex(index=probes_labeled.index)
+ # Check for signs that the wrong reference was used
+ num_missing = pd.isnull(ref_matched.start).sum()
+ if num_missing > 0:
+ raise ValueError("Reference is missing %d bins found in %s"
+ % (num_missing, probes.sample_id))
+ return ref_pset.as_dataframe(ref_matched.reset_index(drop=True))
+
+
+def center_by_window(cnarr, fraction, sort_key):
+ """Smooth out biases according to the trait specified by sort_key.
+
+ E.g. correct GC-biased probes by windowed averaging across similar-GC
+ probes; or for similar interval sizes.
+ """
+ # Separate neighboring probes that could have the same key
+ # (to avoid re-centering actual CNV regions -- only want an independently
+ # sampled subset of presumably overall-CN-neutral probes)
+ df = cnarr.data.reset_index(drop=True)
+ shuffle_order = np.random.permutation(df.index)
+ df = df.reindex(shuffle_order)
+ # Apply the same shuffling to the key array as to the target probe set
+ assert isinstance(sort_key, (np.ndarray, pd.Series))
+ sort_key = sort_key[shuffle_order]
+ # Sort the data according to the specified parameter
+ order = np.argsort(sort_key, kind='mergesort')
+ df = df.iloc[order]
+ biases = smoothing.rolling_median(df['log2'], fraction)
+ # biases = smoothing.smoothed(df['log2'], fraction)
+ df['log2'] -= biases
+ fixarr = cnarr.as_dataframe(df)
+ fixarr.sort()
+ return fixarr
+
+
+def get_edge_bias(cnarr, margin):
+ """Quantify the "edge effect" of the target tile and its neighbors.
+
+ The result is proportional to the change in the target's coverage due to
+ these edge effects, i.e. the expected loss of coverage near the target
+ edges and, if there are close neighboring tiles, gain of coverage due
+ to "spill over" reads from the neighbor tiles.
+
+ (This is not the actual change in coverage. This is just a tribute.)
+ """
+ output_by_chrom = []
+ for _chrom, subarr in cnarr.by_chromosome():
+ tile_starts = np.asarray(subarr['start'])
+ tile_ends = np.asarray(subarr['end'])
+ tgt_sizes = tile_ends - tile_starts
+ # Calculate coverage loss at (both edges of) each tile
+ losses = edge_losses(tgt_sizes, margin)
+ # Find tiled intervals within a margin (+/- bp) of the given probe
+ # (excluding the probe itself), then calculate the relative coverage
+ # "gain" due to the neighbors, if any
+ gap_sizes = np.asarray(tile_starts[1:]) - np.asarray(tile_ends[:-1])
+ ok_gaps_mask = (gap_sizes < margin)
+ ok_gaps = gap_sizes[ok_gaps_mask]
+ left_gains = edge_gains(tgt_sizes[1:][ok_gaps_mask], ok_gaps, margin)
+ right_gains = edge_gains(tgt_sizes[:-1][ok_gaps_mask], ok_gaps, margin)
+ gains = np.zeros(len(subarr))
+ gains[np.concatenate([[False], ok_gaps_mask])] += left_gains
+ gains[np.concatenate([ok_gaps_mask, [False]])] += right_gains
+ output_by_chrom.append(gains - losses)
+ return np.concatenate(output_by_chrom)
+
+
+def edge_losses(target_sizes, insert_size):
+ """Calculate coverage losses at the edges of baited regions.
+
+ Letting i = insert size and t = target size, the proportional loss of
+ coverage near the two edges of the baited region (combined) is::
+
+ i/2t
+
+ If the "shoulders" extend outside the bait $(t < i), reduce by::
+
+ (i-t)^2 / 4it
+
+ on each side, or (i-t)^2 / 2it total.
+ """
+ losses = insert_size / (2 * target_sizes)
+ # Drop the shoulder part that would extend past the bait
+ small_mask = (target_sizes < insert_size)
+ t_small = target_sizes[small_mask]
+ losses[small_mask] -= ((insert_size - t_small)**2
+ / (2 * insert_size * t_small))
+ return losses
+
+
+def edge_gains(target_sizes, gap_sizes, insert_size):
+ """Calculate coverage gain from neighboring baits' flanking reads.
+
+ Letting i = insert size, t = target size, g = gap to neighboring bait,
+ the gain of coverage due to a nearby bait, if g < i, is::
+
+ (i-g)^2 / 4it
+
+ If the neighbor flank extends beyond the target (t+g < i), reduce by::
+
+ (i-t-g)^2 / 4it
+
+ If a neighbor overlaps the target, treat it as adjacent (gap size 0).
+ """
+ if not (gap_sizes <= insert_size).all():
+ raise ValueError("Gaps greater than insert size:\n" +
+ gap_sizes[gap_sizes > insert_size].head())
+ gap_sizes = np.maximum(0, gap_sizes)
+ gains = ((insert_size - gap_sizes)**2
+ / (4 * insert_size * target_sizes))
+ # Drop the flank part that extends past this baited region
+ past_other_side_mask = (target_sizes + gap_sizes < insert_size)
+ g_past = gap_sizes[past_other_side_mask]
+ t_past = target_sizes[past_other_side_mask]
+ gains[past_other_side_mask] -= ((insert_size - t_past - g_past)**2
+ / (4 * insert_size * t_past))
+ return gains
+
+
+def apply_weights(cnarr, ref_matched, epsilon=1e-4):
+ """Calculate weights for each bin.
+
+ Weights are derived from:
+
+ - bin sizes
+ - average bin coverage depths in the reference
+ - the "spread" column of the reference.
+ """
+ # Relative bin sizes
+ sizes = ref_matched['end'] - ref_matched['start']
+ weights = sizes / sizes.max()
+ if (np.abs(np.mod(ref_matched['log2'], 1)) > epsilon).any():
+ # NB: Not used with a flat reference
+ logging.info("Weighting bins by relative coverage depths in reference")
+ # Penalize bins that deviate from neutral coverage
+ flat_cvgs = ref_matched.expect_flat_cvg()
+ weights *= np.exp2(-np.abs(ref_matched['log2'] - flat_cvgs))
+ if (ref_matched['spread'] > epsilon).any():
+ # NB: Not used with a flat or paired reference
+ logging.info("Weighting bins by coverage spread in reference")
+ # Inverse of variance, 0--1
+ variances = ref_matched['spread'] ** 2
+ invvars = 1.0 - (variances / variances.max())
+ weights = (weights + invvars) / 2
+ # Avoid 0-value bins -- CBS doesn't like these
+ weights = np.maximum(weights, epsilon)
+ return cnarr.add_columns(weight=weights)
+
diff --git a/cnvlib/gary.py b/cnvlib/gary.py
new file mode 100644
index 0000000..bd6cf4c
--- /dev/null
+++ b/cnvlib/gary.py
@@ -0,0 +1,490 @@
+"""A generic array of genomic positions."""
+from __future__ import print_function, absolute_import, division
+
+import logging
+import sys
+import warnings
+
+import numpy as np
+import pandas as pd
+
+from . import core, ngfrills
+
+
+class GenomicArray(object):
+ """An array of genomic intervals. Base class for genomic data structures.
+
+ Can represent most BED-like tabular formats with arbitrary additional
+ columns.
+ """
+ _required_columns = ("chromosome", "start", "end")
+ _required_dtypes = ("string", "int", "int")
+
+ def __init__(self, data_table, meta_dict=None):
+ # Validation
+ if len(data_table):
+ if not all(c in data_table.columns for c in self._required_columns):
+ raise ValueError("data table must have at least columns "
+ + repr(self._required_columns))
+ # Ensure chromosomes are strings to avoid integer conversion of 1, 2...
+ if not isinstance(data_table.chromosome.iat[0], basestring):
+ data_table["chromosome"] = (data_table["chromosome"]
+ .astype("string"))
+ elif not isinstance(data_table, pd.DataFrame):
+ # Empty but conformant table
+ data_table = self._make_blank()
+ self.data = data_table
+ self.meta = (dict(meta_dict)
+ if meta_dict is not None and len(meta_dict)
+ else {})
+
+ @staticmethod
+ def row2label(row):
+ return "{}:{}-{}".format(row.chromosome, row.start, row.end)
+
+ @classmethod
+ def _make_blank(cls):
+ """Create an empty dataframe with the columns required by this class."""
+ table = pd.DataFrame({key: [] for key in cls._required_columns})
+ for col, dtype in zip(cls._required_columns, cls._required_dtypes):
+ table[col] = table[col].astype(dtype)
+ return table
+
+ @classmethod
+ def from_columns(cls, columns, meta_dict=None):
+ """Create a new instance from column arrays, given as a dict."""
+ table = pd.DataFrame.from_dict(columns)
+ ary = cls(table, meta_dict)
+ ary.sort_columns()
+ return ary
+
+ @classmethod
+ def from_rows(cls, rows, columns=None, meta_dict=None):
+ """Create a new instance from a list of rows, as tuples or arrays."""
+ if columns is None:
+ columns = cls._required_columns
+ table = pd.DataFrame.from_records(rows, columns=columns)
+ return cls(table, meta_dict)
+
+ def as_columns(self, **columns):
+ """Wrap the named columns in this instance's metadata."""
+ return self.__class__.from_columns(columns, self.meta)
+ # return self.__class__(self.data.loc[:, columns], self.meta.copy())
+
+ def as_dataframe(self, dframe):
+ """Wrap the given pandas dataframe in this instance's metadata."""
+ return self.__class__(dframe.reset_index(drop=True), self.meta.copy())
+
+ # def as_index(self, index):
+ # """Subset with fancy/boolean indexing; reuse this instance's metadata."""
+ # """Extract rows by indices, reusing this instance's metadata."""
+ # if isinstance(index, (int, slice)):
+ # return self.__class__(self.data.iloc[index], self.meta.copy())
+ # else:
+ # return self.__class__(self.data[index], self.meta.copy())
+
+ def as_rows(self, rows):
+ """Wrap the given rows in this instance's metadata."""
+ return self.from_rows(rows,
+ columns=self.data.columns,
+ meta_dict=self.meta)
+
+ # Container behaviour
+
+ def __eq__(self, other):
+ return (isinstance(other, self.__class__) and
+ self.data.equals(other.data))
+
+ def __len__(self):
+ return len(self.data)
+
+ def __contains__(self, key):
+ return key in self.data.columns
+
+ def __getitem__(self, index):
+ """Access a portion of the data.
+
+ Cases:
+
+ - single integer: a row, as pd.Series
+ - string row name: a column, as pd.Series
+ - a boolean array: masked rows, as_dataframe
+ - tuple of integers: selected rows, as_dataframe
+ """
+ if isinstance(index, int):
+ # A single row
+ return self.data.iloc[index]
+ # return self.as_dataframe(self.data.iloc[index:index+1])
+ elif isinstance(index, basestring):
+ # A column, by name
+ return self.data[index]
+ elif (isinstance(index, tuple) and
+ len(index) == 2 and
+ index[1] in self.data.columns):
+ # Row index, column index -> cell value
+ return self.data.loc[index]
+ elif isinstance(index, slice):
+ # return self.as_dataframe(self.data.take(index))
+ return self.as_dataframe(self.data[index])
+ else:
+ # Iterable -- selected row indices or boolean array, probably
+ try:
+ if isinstance(index, type(None)) or len(index) == 0:
+ empty = pd.DataFrame(columns=self.data.columns)
+ return self.as_dataframe(empty)
+ except TypeError:
+ raise TypeError("object of type %r " % type(index) +
+ "cannot be used as an index into a " +
+ self.__class__.__name__)
+ return self.as_dataframe(self.data[index])
+ # return self.as_dataframe(self.data.take(index))
+
+ def __setitem__(self, index, value):
+ """Assign to a portion of the data.
+ """
+ # self.data[index] = value
+ if isinstance(index, int):
+ self.data.iloc[index] = value
+ elif isinstance(index, basestring):
+ self.data[index] = value
+ elif (isinstance(index, tuple) and
+ len(index) == 2 and
+ index[1] in self.data.columns):
+ self.data.loc[index] = value
+ else:
+ assert isinstance(index, slice) or len(index) > 0
+ self.data[index] = value
+
+ def __delitem__(self, index):
+ return NotImplemented
+
+ def __iter__(self):
+ return self.data.itertuples(index=False)
+
+ __next__ = next
+
+ @property
+ def chromosome(self):
+ return self.data['chromosome']
+
+ @property
+ def start(self):
+ return self.data['start']
+
+ @property
+ def end(self):
+ return self.data['end']
+
+ @property
+ def sample_id(self):
+ return self.meta.get('sample_id')
+
+ # Traversal
+
+ def autosomes(self, also=()):
+ """Select chromosomes w/ integer names, ignoring any 'chr' prefixes."""
+ with warnings.catch_warnings():
+ # NB: We're not using the deprecated part of this pandas method
+ warnings.simplefilter("ignore", UserWarning)
+ is_auto = self.chromosome.str.match(r"(chr)?\d+$",
+ as_indexer=True, na=False)
+ if not is_auto.any():
+ # The autosomes, if any, are not named with plain integers
+ return self
+ if also:
+ if isinstance(also, basestring):
+ also = [also]
+ for a_chrom in also:
+ is_auto |= (self.chromosome == a_chrom)
+ return self[is_auto]
+
+ def by_chromosome(self):
+ """Iterate over bins grouped by chromosome name."""
+ for chrom, subtable in self.data.groupby("chromosome", sort=False):
+ yield chrom, self.as_dataframe(subtable)
+
+ def by_ranges(self, other, mode='inner', keep_empty=True):
+ """Group rows by another GenomicArray's bin coordinate ranges.
+
+ Returns an iterable of (bin, GenomicArray of overlapping rows)). Usually
+ used for grouping probes or SNVs by CNV segments.
+
+ `mode` determines what to do with bins that overlap a boundary of the
+ selection. Values are:
+
+ - ``inner``: Drop the bins on the selection boundary, don't emit them.
+ - ``outer``: Keep/emit those bins as they are.
+ - ``trim``: Emit those bins but alter their boundaries to match the
+ selection; the bin start or end position is replaced with the
+ selection boundary position.
+
+ Bins in this array that fall outside the other array's bins are skipped.
+ """
+ chrom_lookup = dict(self.by_chromosome())
+ for chrom, bin_rows in other.by_chromosome():
+ if chrom in chrom_lookup:
+ subranges = chrom_lookup[chrom]._iter_ranges(
+ None, bin_rows['start'], bin_rows['end'], mode)
+ for bin_row, subrange in zip(bin_rows, subranges):
+ yield bin_row, subrange
+ else:
+ if keep_empty:
+ for bin_row in bin_rows:
+ yield bin_row, self.as_rows([])
+
+ def coords(self, also=()):
+ """Iterate over plain coordinates of each bin: chromosome, start, end.
+
+ With `also`, also include those columns.
+
+ Example, yielding rows in BED format:
+
+ >>> probes.coords(also=["name", "strand"])
+ """
+ cols = list(GenomicArray._required_columns)
+ if also:
+ if isinstance(also, basestring):
+ cols.append(also)
+ else:
+ cols.extend(also)
+ coordframe = self.data.loc[:, cols]
+ return coordframe.itertuples(index=False)
+
+ def labels(self):
+ return self.data.apply(self.row2label, axis=1)
+
+ def in_range(self, chrom=None, start=None, end=None, mode='inner'):
+ """Get the GenomicArray portion within the given genomic range.
+
+ `mode` works as in `by_ranges`: ``outer`` includes bins straddling the
+ range boundaries, ``trim`` additionally alters the straddling bins'
+ endpoints to match the range boundaries, and ``inner`` excludes those
+ bins.
+ """
+ if isinstance(start, (int, float, np.float64)):
+ start = [int(start)]
+ if isinstance(end, (int, float, np.float64)):
+ end = [int(end)]
+ results = self._iter_ranges(chrom, start, end, mode)
+ return next(results)
+
+ def in_ranges(self, chrom=None, starts=None, ends=None, mode='inner'):
+ """Get the GenomicArray portion within the specified ranges.
+
+ Same as `in_ranges` but the `starts` and `ends` are arrays of equal
+ length, and the output concatenates all the selected bins.
+ """
+ return self.concat(self._iter_ranges(chrom, starts, ends, mode))
+
+ def _iter_ranges(self, chrom, starts, ends, mode):
+ """Iterate through sub-ranges."""
+ assert mode in ('inner', 'outer', 'trim')
+ if chrom:
+ assert isinstance(chrom, basestring) # ENH: accept array?
+ try:
+ table = self.data[self.data['chromosome'] == chrom]
+ except KeyError:
+ raise KeyError("Chromosome %s is not in this probe set" % chrom)
+ else:
+ # Unsafe, but faster if we've already subsetted by chromosome
+ table = self.data
+
+ # Edge cases
+ if not len(table):
+ yield self.as_rows([])
+ raise StopIteration
+
+ if starts is None and ends is None:
+ yield self.as_dataframe(table)
+ raise StopIteration
+
+ if starts is not None and len(starts):
+ if mode == 'inner':
+ # Only rows entirely after the start point
+ start_idxs = table.start.searchsorted(starts)
+ else:
+ # Include all rows overlapping the start point
+ start_idxs = table.end.searchsorted(starts, 'right')
+ else:
+ starts = np.zeros(len(ends) if ends is not None else 1,
+ dtype=np.int_)
+ start_idxs = starts.copy()
+
+ if ends is not None and len(ends):
+ if mode == 'inner':
+ end_idxs = table.end.searchsorted(ends, 'right')
+ else:
+ end_idxs = table.start.searchsorted(ends)
+ else:
+ end_idxs = np.repeat(len(table), len(starts))
+ ends = [None] * len(starts)
+
+ for start_idx, start_val, end_idx, end_val in zip(start_idxs, starts,
+ end_idxs, ends):
+ subtable = table[start_idx:end_idx]
+ if mode == 'trim':
+ subtable = subtable.copy()
+ # Update 5' endpoints to the boundary
+ if start_val:
+ subtable.start = subtable.start.clip_lower(start_val)
+ # Update 3' endpoints to the boundary
+ if end_val:
+ subtable.end = subtable.end.clip_upper(end_val)
+ yield self.as_dataframe(subtable)
+
+ def match_to_bins(self, other, key, default=0.0, fill=False,
+ summary_func=np.median):
+ """Take values of the other array at each of this array's bins.
+
+ Assign `default` to indices that fall outside the other array's bins, or
+ chromosomes that appear in `self` but not `other`.
+
+ Return an array of the `key` column values in `other` corresponding to this
+ array's bin locations, the same length as this array.
+ """
+ def rows2value(rows):
+ if len(rows) == 0:
+ return default
+ elif len(rows) == 1:
+ return rows[0, key]
+ else:
+ return summary_func(rows[key])
+
+ all_out_vals = [rows2value(other_rows) for _bin, other_rows in
+ other.by_ranges(self, mode='outer', keep_empty=True)]
+ return np.asarray(all_out_vals)
+
+ # Modification
+
+ def add(self, other):
+ """Combine this array's data with another GenomicArray (in-place).
+
+ Any optional columns must match between both arrays.
+ """
+ if not isinstance(other, self.__class__):
+ raise ValueError("Argument (type %s) is not a %s instance"
+ % (type(other), self.__class__))
+ if not len(other.data):
+ return self.copy()
+ self.data = pd.concat([self.data, other.data])
+ self.sort()
+
+ def concat(self, others):
+ """Concatenate several GenomicArrays, keeping this array's metadata.
+
+ This array's data table is not implicitly included in the result.
+ """
+ result = self.as_dataframe(pd.concat([otr.data for otr in others]))
+ result.sort()
+ return result
+
+ def copy(self):
+ """Create an independent copy of this object."""
+ return self.as_dataframe(self.data.copy())
+
+ def add_columns(self, **columns):
+ """Create a new CNA, adding the specified extra columns to this CNA."""
+ # return self.as_dataframe(self.data.assign(**columns))
+ result = self.copy()
+ for key, values in columns.iteritems():
+ result[key] = values
+ return result
+
+ def keep_columns(self, columns):
+ """Extract a subset of columns, reusing this instance's metadata."""
+ return self.__class__(self.data.loc[:, columns], self.meta.copy())
+
+ def drop_extra_columns(self):
+ """Remove any optional columns from this GenomicArray.
+
+ Returns a new copy with only the core columns retained:
+ log2 value, chromosome, start, end, bin name.
+ """
+ table = self.data.loc[:, self._required_columns]
+ return self.as_dataframe(table)
+
+ def select(self, selector=None, **kwargs):
+ """Take a subset of rows where the given condition is true.
+
+ Arguments can be a function (lambda expression) returning a bool, which
+ will be used to select True rows, and/or keyword arguments like
+ gene="Background" or chromosome="chr7", which will select rows where the
+ keyed field equals the specified value.
+ """
+ table = self.data
+ if selector is not None:
+ table = table[table.apply(selector, axis=1)]
+ for key, val in kwargs.items():
+ assert key in self
+ table = table[table[key] == val]
+ return self.as_dataframe(table)
+
+ def shuffle(self):
+ """Randomize the order of bins in this array (in-place)."""
+ np.random.seed(0xA5EED) # For reproducible results
+ order = np.arange(len(self.data))
+ np.random.shuffle(order)
+ self.data = self.data.iloc[order]
+ return order
+
+ def sort(self):
+ """Sort this array's bins in-place, with smart chromosome ordering."""
+ table = self.data.copy()
+ table['SORT_KEY'] = self.chromosome.apply(core.sorter_chrom)
+ table.sort_values(by=['SORT_KEY', 'start'], inplace=True)
+ del table['SORT_KEY']
+ self.data = table.reset_index(drop=True)
+
+ def sort_columns(self):
+ """Sort this array's columns in-place, per class definition."""
+ extra_cols = []
+ for col in self.data.columns:
+ if col not in self._required_columns:
+ extra_cols.append(col)
+ sorted_colnames = list(self._required_columns) + sorted(extra_cols)
+ assert len(sorted_colnames) == len(self.data.columns)
+ self.data = self.data.reindex(columns=sorted_colnames)
+
+ # I/O
+
+ @classmethod
+ def read(cls, infile, sample_id=None):
+ if sample_id is None:
+ if isinstance(infile, basestring):
+ sample_id = core.fbase(infile)
+ else:
+ sample_id = '<unknown>'
+ try:
+ table = pd.read_table(infile,
+ dtype={'chromosome': 'string'})
+ if "log2" in table.columns:
+ # Every bin needs a log2 value; the others can be NaN
+ t2 = table.dropna(subset=["log2"])
+ if len(t2) < len(table):
+ logging.warn("Dropped %d rows with missing log2 values",
+ len(table) - len(t2))
+ table = t2
+ except ValueError:
+ # File is blank/empty, most likely
+ logging.warn("Blank file?: %s", infile)
+ table = cls._make_blank()
+ # ENH
+ # table['chromosome'] = pd.Categorical(table['chromosome'],
+ # table.chromosome.drop_duplicates(),
+ # ordered=True)
+ # Create a multi-index of genomic coordinates (like GRanges)
+ # table.set_index(['chromosome', 'start'], inplace=True)
+ return cls(table, {"sample_id": sample_id})
+
+ def write(self, outfile=None):
+ """Write the wrapped data table to a file or handle in tabular format.
+
+ The format is BED-like, but with a header row included and with
+ arbitrary extra columns.
+
+ To combine multiple samples in one file and/or convert to another
+ format, see the 'export' subcommand.
+ """
+ with ngfrills.safe_write(outfile or sys.stdout) as handle:
+ self.data.to_csv(handle, index=False, sep='\t', float_format='%.6g')
+
diff --git a/cnvlib/importers.py b/cnvlib/importers.py
new file mode 100644
index 0000000..5b46ffc
--- /dev/null
+++ b/cnvlib/importers.py
@@ -0,0 +1,208 @@
+"""Import from other formats to the CNVkit format."""
+from __future__ import absolute_import, division, print_function
+
+import logging
+import math
+import os.path
+import subprocess
+
+import numpy as np
+import pandas as pd
+
+from . import core, params
+from .cnary import CopyNumArray as CNA
+
+
+# __________________________________________________________________________
+# import-picard
+
+TOO_MANY_NO_COVERAGE = 100
+
+def find_picard_files(file_and_dir_names):
+ """Search the given paths for 'targetcoverage' CSV files.
+
+ Per the convention we use in our Picard applets, the target coverage file
+ names end with '.targetcoverage.csv'; anti-target coverages end with
+ '.antitargetcoverage.csv'.
+ """
+ filenames = []
+ for tgt in file_and_dir_names:
+ if os.path.isdir(tgt):
+ # Collect the target coverage files from this directory tree
+ fnames = subprocess.check_output(['find', tgt,
+ '-name', '*targetcoverage.csv']
+ ).splitlines()
+ if not fnames:
+ raise RuntimeError("Given directory %s does not contain any "
+ "'*targetcoverage.csv' files."
+ % tgt)
+ filenames.extend(fnames)
+ elif os.path.isfile(tgt):
+ filenames.append(tgt)
+ else:
+ raise ValueError("Given path is neither a file nor a directory: %s"
+ % tgt)
+ filenames.sort()
+ return filenames
+
+
+def import_picard_pertargetcoverage(fname):
+ """Parse a Picard CalculateHsMetrics PER_TARGET_COVERAGE file.
+
+ Return a CopyNumArray.
+
+ Input column names:
+ chrom (str),
+ start, end, length (int),
+ name (str),
+ %gc, mean_coverage, normalized_coverage (float)
+ """
+ dframe = pd.read_table(fname, na_filter=False)
+ coverages = np.asarray(dframe['mean_coverage'])
+ no_cvg_idx = (coverages == 0)
+ if sum(no_cvg_idx) > TOO_MANY_NO_COVERAGE:
+ logging.warn("*WARNING* Sample %s has >%d bins with no coverage",
+ fname, TOO_MANY_NO_COVERAGE)
+ # Avoid math domain error
+ coverages[no_cvg_idx] = 2**params.NULL_LOG2_COVERAGE
+ cnarr = CNA.from_columns({"chromosome": dframe["chrom"],
+ "start": dframe["start"] - 1,
+ "end": dframe["end"],
+ "gene": dframe["name"].apply(unpipe_name),
+ "gc": dframe["%gc"],
+ "log2": np.log2(coverages)},
+ {"sample_id": core.fbase(fname)})
+ cnarr.sort()
+ return cnarr
+
+
+def unpipe_name(name):
+ """Fix the duplicated gene names Picard spits out.
+
+ Return a string containing the single gene name, sans duplications and pipe
+ characters.
+
+ Picard CalculateHsMetrics combines the labels of overlapping intervals
+ by joining all labels with '|', e.g. 'BRAF|BRAF' -- no two distinct
+ targeted genes actually overlap, though, so these dupes are redundant.
+ Meaningless target names are dropped, e.g. 'CGH|FOO|-' resolves as 'FOO'.
+ In case of ambiguity, the longest name is taken, e.g. "TERT|TERT Promoter"
+ resolves as "TERT Promoter".
+ """
+ if '|' not in name:
+ return name
+ gene_names = set(name.split('|'))
+ if len(gene_names) == 1:
+ return gene_names.pop()
+ cleaned_names = gene_names.difference(params.IGNORE_GENE_NAMES)
+ if cleaned_names:
+ gene_names = cleaned_names
+ new_name = sorted(gene_names, key=len, reverse=True)[0]
+ if len(gene_names) > 1:
+ logging.warn("*WARNING* Ambiguous gene name %r; using %r",
+ name, new_name)
+ return new_name
+
+
+# __________________________________________________________________________
+# import-seg
+
+LOG2_10 = math.log(10, 2) # To convert log10 values to log2
+
+def import_seg(segfname, chrom_names, chrom_prefix, from_log10):
+ """Parse a SEG file as an iterable of CopyNumArray instances.
+
+ `chrom_names`:
+ Map (string) chromosome IDs to names. (Applied before chrom_prefix.)
+ e.g. {'23': 'X', '24': 'Y', '25': 'M'}
+
+ `chrom_prefix`: prepend this string to chromosome names
+ (usually 'chr' or None)
+
+ `from_log10`: Convert values from log10 to log2.
+ """
+ dframe = pd.read_table(segfname, na_filter=False)
+ if len(dframe.columns) == 6:
+ dframe.columns = ['sample_id', 'chromosome', 'start', 'end', 'nprobes',
+ 'mean']
+ elif len(dframe.columns) == 5:
+ dframe.columns = ['sample_id', 'chromosome', 'start', 'end', 'mean']
+ else:
+ raise ValueError("SEG format expects 5 or 6 columns; found {}: {}"
+ .format(len(dframe.columns), ' '.join(dframe.columns)))
+
+ # Calculate values for output columns
+ dframe['chromosome'] = dframe['chromosome'].apply(str)
+ if chrom_names:
+ dframe['chromosome'] = dframe['chromosome'].apply(lambda c:
+ chrom_names.get(c, c))
+ if chrom_prefix:
+ dframe['chromosome'] = dframe['chromosome'].apply(lambda c:
+ chrom_prefix + c)
+ if from_log10:
+ dframe['mean'] *= LOG2_10
+ dframe['gene'] = ["G" if mean >= 0 else "L" for mean in dframe['mean']]
+
+ for sid in pd.unique(dframe['sample_id']):
+ sample = dframe[dframe['sample_id'] == sid]
+ cols = {'chromosome': sample['chromosome'],
+ 'start': sample['start'],
+ 'end': sample['end'],
+ 'gene': sample['gene'],
+ 'log2': sample['mean']}
+ if 'nprobes' in dframe:
+ cols['probes'] = sample['nprobes']
+ cns = CNA.from_columns(cols, {'sample_id': sid})
+ cns.sort()
+ yield cns
+
+
+# __________________________________________________________________________
+# import-theta
+
+def parse_theta_results(fname):
+ """Parse THetA results into a data structure.
+
+ Columns: NLL, mu, C, p*
+ """
+ with open(fname) as handle:
+ header = next(handle).rstrip().split('\t')
+ body = next(handle).rstrip().split('\t')
+ assert len(body) == len(header) == 4
+
+ # NLL
+ nll = float(body[0])
+
+ # mu
+ mu = body[1].split(',')
+ mu_normal = float(mu[0])
+ mu_tumors = map(float, mu[1:])
+
+ # C
+ copies = body[2].split(':')
+ if len(mu_tumors) == 1:
+ # 1D array of integers
+ # Replace X with None for "missing"
+ copies = [[int(c) if c.isdigit() else None
+ for c in copies]]
+ else:
+ # List of lists of integer-or-None (usu. 2 x #segments)
+ copies = [[int(c) if c.isdigit() else None
+ for c in subcop]
+ for subcop in zip(*[c.split(',') for c in copies])]
+
+ # p*
+ probs = body[3].split(',')
+ if len(mu_tumors) == 1:
+ # 1D array of floats, or None for "X" (missing/unknown)
+ probs = [float(p) if not p.isalpha() else None
+ for p in probs]
+ else:
+ probs = [[float(p) if not p.isalpha() else None
+ for p in subprob]
+ for subprob in zip(*[p.split(',') for p in probs])]
+ return {"NLL": nll,
+ "mu_normal": mu_normal,
+ "mu_tumors": mu_tumors,
+ "C": copies,
+ "p*": probs}
diff --git a/cnvlib/jenks.py b/cnvlib/jenks.py
new file mode 100644
index 0000000..b91591f
--- /dev/null
+++ b/cnvlib/jenks.py
@@ -0,0 +1,134 @@
+#!/usr/bin/env python
+
+"""Jenks natural breaks optimization.
+
+See:
+
+- https://en.wikipedia.org/wiki/Jenks_natural_breaks_optimization
+- https://www.macwright.org/2013/02/18/literate-jenks.html
+- https://www.macwright.org/simple-statistics/docs/simple_statistics.html
+"""
+from __future__ import division, print_function
+
+import numpy as np
+
+
+def jenks(data, n_classes):
+ """Find breakpoints using Jenks natural breaks optimization.
+
+ NB: The `data` array is sorted in-place.
+ """
+ assert n_classes <= len(data)
+ data.sort()
+ lower_class_limits, _variance_combinations = jenks_matrices(data, n_classes)
+ # print(variance_combinations)
+
+ # Extract n_classes out of the computed matrices
+ breakpoints = jenks_breaks(data, lower_class_limits, n_classes)
+ return breakpoints
+
+
+def jenks_matrices(data, n_classes):
+ """Compute Matrices for Jenks.
+
+ Compute the matrices required for Jenks breaks. These matrices can be used
+ for any classing of data with `classes <= n_classes`.
+ """
+ n_rows = len(data) + 1
+ n_cols = n_classes + 1
+ # Optimal lower class limits
+ lower_class_limits = np.zeros((n_rows, n_cols),
+ dtype=np.float_)
+ lower_class_limits[1,1:] = 1.0
+ lower_class_limits[2:,1] = 1.0
+ # Looks like e.g. if n_classes=3, len(data)=5:
+ # 0 0 0 0
+ # 0 1 1 1
+ # 0 1 0 0
+ # 0 1 0 0
+ # 0 1 0 0
+ # 0 1 0 0
+
+ # Optimal variance combinations for all classes
+ variance_combinations = np.zeros((n_rows, n_cols),
+ dtype=np.float_)
+ # variance_combinations[1,1:] = 0.0
+ variance_combinations[2:,1:] = float('inf')
+ # e.g.:
+ # 0 0 0 0
+ # 0 0 0 0
+ # 0 i i i
+ # 0 i i i
+ # 0 i i i
+ # 0 i i i
+
+ for l in xrange(2, n_rows):
+ # Estimate variance for each potential classing of the data, for
+ # each potential number of classes.
+ # ENH: lift these out of the loop & apply more array magic
+ vals = data[np.arange(l - 1, -1, -1)]
+ sums = np.cumsum(vals)
+ sums_square = np.cumsum(vals ** 2)
+ variances = [(sums_square[i] - (sums[i] ** 2)) / (i+1)
+ for i in range(l)]
+ # Record the last variance in column 1
+ variance_combinations[l][1] = variances[l-1]
+
+ for m in xrange(1, l):
+ # For each column from index 1 to just before the diagonal
+ # Or, for each element up to the current l marker again
+ lower_class_limit = l - m + 1 # >=2, <=l
+ iv = lower_class_limit - 1 # >=1, <l
+ variance = variances[m-1]
+ for j in xrange(2, n_cols):
+ # If adding this element to an existing class will increase
+ # its variance beyond the limit, break the class at this
+ # point, setting the `lower_class_limit` at this point.
+ var_up_left = variance + variance_combinations[iv,j-1]
+ if variance_combinations[l,j] >= var_up_left:
+ lower_class_limits[l,j] = lower_class_limit
+ variance_combinations[l,j] = var_up_left
+
+ # Return the two matrices.
+ # Only `lower_class_limits` is needed to calculate breaks, but
+ # variances can be useful to evaluate goodness of fit.
+ return lower_class_limits, variance_combinations
+
+
+def jenks_breaks(data, lower_class_limits, n_classes):
+ """Pull Breaks Values for Jenks.
+
+ The second part of the jenks recipe: Take the calculated matrices and derive
+ an array of n breaks.
+
+ Backtracking, in DP lingo.
+ """
+ breakpoints = np.zeros(n_classes + 1)
+ breakpoints[n_classes] = data[-1] # Upper bound
+ # Use the lower_class_limits matrix as indices into itself, iteratively
+ lower_limit_idx = len(data)
+ for j in xrange(n_classes, 0, -1):
+ lower_limit_idx = lower_class_limits[lower_limit_idx, j] - 1
+ breakpoints[j - 1] = data[lower_limit_idx]
+ return breakpoints
+
+
+if __name__ == '__main__':
+ import sys
+ if len(sys.argv) == 2:
+ a = np.loadtxt(sys.argv[1])
+ else:
+ np.random.seed(0x5EED)
+ a = np.concatenate([
+ np.random.randn(600)*1.5 - 4,
+ np.random.randn(400)*.5 - 1,
+ np.random.randn(900) + 2])
+ print("Data range:", a.min(), a.max())
+ bp = jenks(a, 3)
+ print("Breakpoints:", *bp)
+ lower = a[a < bp[1]]
+ middle = a[(bp[1] <= a) & (a <= bp[2])]
+ upper = a[bp[2] < a]
+ print("Set sizes:", len(lower), len(middle), len(upper))
+ print("Total:", sum([len(lower), len(middle), len(upper)]))
+
diff --git a/cnvlib/metrics.py b/cnvlib/metrics.py
new file mode 100644
index 0000000..76b790f
--- /dev/null
+++ b/cnvlib/metrics.py
@@ -0,0 +1,200 @@
+"""Robust estimators of central tendency and scale.
+
+For use in evaluating performance of copy number estimation.
+
+See:
+ https://en.wikipedia.org/wiki/Robust_measures_of_scale
+ https://astropy.readthedocs.org/en/latest/_modules/astropy/stats/funcs.html
+
+"""
+from __future__ import division
+
+import numpy as np
+from scipy import stats
+
+
+def ests_of_scale(deviations):
+ """Estimators of scale: standard deviation, MAD, biweight midvariance.
+
+ Calculates all of these values for an array of deviations and returns them
+ as a tuple.
+ """
+ std = np.std(deviations, dtype=np.float64)
+ mad = median_absolute_deviation(deviations)
+ iqr = interquartile_range(deviations)
+ biw = biweight_midvariance(deviations)
+ return (std, mad, iqr, biw)
+
+
+# M-estimators of central location
+
+def biweight_location(a, initial=None, c=6.0, epsilon=1e-4):
+ """Compute the biweight location for an array.
+
+ The biweight is a robust statistic for determining the central location of a
+ distribution.
+ """
+ a = np.asarray(a)
+ if initial is None:
+ initial = np.median(a)
+ # Weight the observations by distance from initial estimate
+ d = a - initial
+ w = d / max(c * median_absolute_deviation(a), epsilon)
+ w = (1 - w**2)**2
+ # Omit the outlier points
+ mask = (w < 1)
+ weightsum = w[mask].sum()
+ if weightsum == 0:
+ # Insufficient variation to improve the initial estimate
+ return initial
+ return initial + (d[mask] * w[mask]).sum() / weightsum
+
+
+def modal_location(arr):
+ """Return the modal value of an array's values.
+
+ The "mode" is the location of peak density among the values, estimated using
+ a Gaussian kernel density estimator.
+
+ `arr` is a 1-D array of floating-point values, e.g. bin log2 ratio values.
+ """
+ sarr = np.sort(arr)
+ kde = stats.gaussian_kde(sarr)
+ y = kde.evaluate(sarr)
+ peak = sarr[y.argmax()]
+ return peak
+
+
+def segment_mean(cnarr, skip_low=False):
+ """Weighted average of bin log2 values."""
+ if skip_low:
+ cnarr = cnarr.drop_low_coverage()
+ if len(cnarr) == 0:
+ return None
+ if 'weight' in cnarr:
+ return np.average(cnarr['log2'], weights=cnarr['weight'])
+ return cnarr['log2'].mean()
+
+
+# Estimators of scale
+
+def biweight_midvariance(a, initial=None, c=9.0, epsilon=1e-4):
+ """Compute the biweight midvariance for an array.
+
+ The biweight midvariance is a robust statistic for determining the
+ midvariance (i.e. the standard deviation) of a distribution.
+
+ See:
+ https://en.wikipedia.org/wiki/Robust_measures_of_scale#The_biweight_midvariance
+ https://astropy.readthedocs.org/en/latest/_modules/astropy/stats/funcs.html
+ """
+ a = np.asarray(a)
+ if initial is None:
+ initial = np.median(a)
+ # Difference of observations from initial estimate
+ d = a - initial
+ # Weighting (avoid dividing by zero)
+ w = d / max(c * median_absolute_deviation(a), epsilon)
+ w = w**2
+ # Omit the outlier points
+ mask = np.abs(w) < 1
+ n = mask.sum()
+ return (n**0.5 * (d[mask] * d[mask] * (1 - w[mask])**4).sum()**0.5
+ / np.abs(((1 - w[mask]) * (1 - 5 * w[mask])).sum()))
+
+
+def interquartile_range(a):
+ """Compute the difference between the array's first and third quartiles."""
+ a = np.asarray(a)
+ return np.percentile(a, 75) - np.percentile(a, 25)
+
+
+def median_absolute_deviation(a, scale_to_sd=True):
+ """Compute the median absolute deviation (MAD) of array elements.
+
+ The MAD is defined as: ``median(abs(a - median(a)))``.
+
+ See: https://en.wikipedia.org/wiki/Median_absolute_deviation
+ """
+ a = np.asarray(a)
+ a_median = np.median(a)
+ mad = np.median(np.abs(a - a_median))
+ if scale_to_sd:
+ mad *= 1.4826
+ return mad
+
+
+def q_n(a):
+ """Rousseeuw & Croux's (1993) Q_n, an alternative to MAD.
+
+ ``Qn := Cn first quartile of (|x_i - x_j|: i < j)``
+
+ where Cn is a constant depending on n.
+
+ Finite-sample correction factors must be used to calibrate the
+ scale of Qn for small-to-medium-sized samples.
+
+ n E[Qn]
+ -- -----
+ 10 1.392
+ 20 1.193
+ 40 1.093
+ 60 1.064
+ 80 1.048
+ 100 1.038
+ 200 1.019
+
+ """
+ a = np.asarray(a)
+
+ # First quartile of: (|x_i - x_j|: i < j)
+ vals = []
+ for i, x_i in enumerate(a):
+ for x_j in a[i+1:]:
+ vals.append(abs(x_i - x_j))
+ quartile = np.percentile(vals, 25)
+
+ # Cn: a scaling factor determined by sample size
+ n = len(a)
+ if n <= 10:
+ # ENH: warn when extrapolating beyond the data
+ # ENH: simulate for values up to 10
+ # (unless the equation below is reliable)
+ scale = 1.392
+ elif 10 < n < 400:
+ # I fitted the simulated values (above) to a power function in Excel:
+ # f(x) = 1.0 + 3.9559 * x ^ -1.0086
+ # This should be OK for interpolation. (Does it apply generally?)
+ scale = 1.0 + (4 / n)
+ else:
+ scale = 1.0
+
+ return quartile / scale
+
+
+# Intervals
+
+def confidence_interval_bootstrap(bins, alpha, bootstraps=100):
+ """Confidence interval for segment mean log2 value, estimated by bootstrap."""
+ # Bootstrap for CI
+ k = len(bins)
+ rand_indices = np.random.random_integers(0, k - 1, (bootstraps, k))
+ bootstraps = [bins.data.take(idx) for idx in rand_indices]
+ # Recalculate segment means
+ if 'weight' in bins:
+ bootstrap_dist = [np.average(boot['log2'], weights=boot['weight'])
+ for boot in bootstraps]
+ else:
+ bootstrap_dist = [boot['log2'].mean() for boot in bootstraps]
+ pct_lo = 100 * alpha / 2
+ pct_hi = 100 * (1 - alpha / 2)
+ return np.percentile(bootstrap_dist, [pct_lo, pct_hi])
+
+
+def prediction_interval(bins, alpha):
+ """Prediction interval, estimated by percentiles."""
+ pct_lo = 100 * alpha / 2
+ pct_hi = 100 * (1 - alpha / 2)
+ # ENH: weighted percentile
+ return np.percentile(bins['log2'], [pct_lo, pct_hi])
+
diff --git a/cnvlib/ngfrills/__init__.py b/cnvlib/ngfrills/__init__.py
new file mode 100644
index 0000000..e65d089
--- /dev/null
+++ b/cnvlib/ngfrills/__init__.py
@@ -0,0 +1,113 @@
+"""NGS utilities."""
+from __future__ import absolute_import, division, print_function
+
+import contextlib
+import logging
+import os
+import subprocess
+import sys
+import tempfile
+
+from Bio._py3k import basestring, map
+
+from .faidx import *
+from .regions import *
+from .samutil import *
+from .shared import *
+
+
+# __________________________________________________________
+# Shell
+
+def call_quiet(*args):
+ """Safely run a command and get stdout; print stderr if there's an error.
+
+ Like subprocess.check_output, but silent in the normal case where the
+ command logs unimportant stuff to stderr. If there is an error, then the
+ full error message(s) is shown in the exception message.
+ """
+ # args = map(str, args)
+ if not len(args):
+ raise ValueError("Must supply at least one argument (the command name)")
+ try:
+ proc = subprocess.Popen(args, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ except OSError as exc:
+ raise RuntimeError("Could not find the executable %r" % args[0]
+ + " -- is it installed correctly?"
+ + "\n(Original error: %s)" % exc)
+ out, err = proc.communicate()
+ if proc.returncode != 0:
+ raise RuntimeError("Subprocess command failed:\n$ %s\n\n%s"
+ % (' '.join(args), err))
+ return out
+
+
+def ensure_path(fname):
+ """Create dirs and move an existing file to avoid overwriting, if necessary.
+
+ If a file already exists at the given path, it is renamed with an integer
+ suffix to clear the way.
+ """
+ if '/' in os.path.normpath(fname):
+ # Ensure the output directory exists
+ dname = os.path.dirname(os.path.abspath(fname))
+ if dname and not os.path.isdir(dname):
+ try:
+ os.makedirs(dname)
+ except OSError as exc:
+ raise OSError("Output path " + fname +
+ " contains a directory " + dname +
+ " that cannot be created: " + str(exc))
+ if os.path.isfile(fname):
+ # Add an integer suffix to the existing file name
+ cnt = 1
+ bak_fname = "%s.%d" % (fname, cnt)
+ while os.path.isfile(bak_fname):
+ cnt += 1
+ bak_fname = "%s.%d" % (fname, cnt)
+ os.rename(fname, bak_fname)
+ logging.info("Moved existing file %s -> %s", fname, bak_fname)
+ return True
+
+
+ at contextlib.contextmanager
+def safe_write(outfile, verbose=True):
+ """Write to a filename or file-like object with error handling.
+
+ If given a file name, open it. If the path includes directories that don't
+ exist yet, create them. If given a file-like object, just pass it through.
+ """
+ if isinstance(outfile, basestring):
+ dirname = os.path.dirname(outfile)
+ if dirname and not os.path.isdir(dirname):
+ os.mkdir(dirname)
+ logging.info("Created directory %s", dirname)
+ with open(outfile, 'w') as handle:
+ yield handle
+ else:
+ yield outfile
+
+ # Log the output path, if possible
+ if verbose:
+ if isinstance(outfile, basestring):
+ outfname = outfile
+ elif hasattr(outfile, 'name') and outfile not in (sys.stdout,
+ sys.stderr):
+ outfname = outfile.name
+ else:
+ # Probably stdout or stderr -- don't ruin the pipeline
+ return
+ logging.info("Wrote %s", outfname)
+
+
+ at contextlib.contextmanager
+def temp_write_text(text):
+ """Save text to a temporary file.
+
+ NB: This won't work on Windows b/c the file stays open.
+ """
+ with tempfile.NamedTemporaryFile() as tmp:
+ tmp.write(text)
+ tmp.flush()
+ yield tmp.name
diff --git a/cnvlib/ngfrills/faidx.py b/cnvlib/ngfrills/faidx.py
new file mode 100644
index 0000000..e2109a2
--- /dev/null
+++ b/cnvlib/ngfrills/faidx.py
@@ -0,0 +1,34 @@
+"""NGS utilities: Indexed FASTA I/O."""
+from __future__ import absolute_import, division, print_function
+
+import logging
+from itertools import groupby
+from pyfaidx import Fasta
+
+
+def fasta_extract_regions(fa_fname, intervals):
+ """Extract an iterable of regions from an indexed FASTA file.
+
+ Input: FASTA file name; iterable of (seq_id, start, end) (1-based)
+ Output: iterable of string sequences.
+ """
+ with Fasta(fa_fname, as_raw=True) as fa_file:
+ for chrom, rows in groupby(intervals, lambda cse: cse[0]):
+ logging.info("Extracting sequences from chromosome %s", chrom)
+ for _chrom, start, end in rows:
+ yield fa_file[_chrom][start:end]
+
+
+def _fasta_extract_regions_safe(fa_fname, intervals):
+ """Simpler, slower version of fasta_extract_regions, for testing it."""
+ from Bio import SeqIO
+ idx = SeqIO.index(fa_fname, 'fasta')
+ for chrom, rows in groupby(intervals, lambda cse: cse[0]):
+ logging.info("Extracting sequences from chromosome %s", chrom)
+ seq = str(idx[chrom].seq)
+ for chrom, start, end in rows:
+ subseq = seq[start:end]
+ if len(subseq) != end - start:
+ logging.info("Short subsequence %s:%d-%d; read %d, wanted %d",
+ chrom, start, end, len(subseq), end - start)
+ yield subseq
diff --git a/cnvlib/ngfrills/regions.py b/cnvlib/ngfrills/regions.py
new file mode 100644
index 0000000..9d6b4ba
--- /dev/null
+++ b/cnvlib/ngfrills/regions.py
@@ -0,0 +1,248 @@
+"""NGS utilities: BED/Interval I/O, genomic regions and ranges."""
+from __future__ import absolute_import, division, print_function
+
+import functools
+import logging
+import re
+import shlex
+
+from Bio.File import as_handle
+
+
+def sniff_num_columns(bed_fname):
+ """Detect the number of columns in a BED/interval file.
+
+ Guidance:
+ 3 cols => coordinates only;
+ 5 cols => intervals file (coordinates, strand, name);
+ otherwise => Full or extended BED format
+ """
+ for firstrow in parse_regions(bed_fname):
+ return len(firstrow)
+
+
+def sniff_region_format(fname):
+ """Guess whether the file format is BED, Picard interval list, or text.
+
+ Returns a tuple of the format name (str) or None if the file is empty.
+ """
+ with open(fname, 'rU') as handle:
+ for line in handle:
+ if not line.strip():
+ continue
+ if '\t' not in line and ':' in line and '-' in line:
+ return 'text'
+ if line.startswith('@') or re.match('\w+\t\d+\t\d+\t(\+|-|\.)\t\S+',
+ line):
+ return 'interval'
+ if line.startswith('track') or line.count('\t') > 1:
+ return 'bed'
+ raise ValueError("File " + repr(fname) + " does not appear to "
+ + "be BED, interval list, or 'chr:start-end' "
+ + "text!\nFirst non-blank line: " + repr(line))
+
+
+def parse_regions(fname, coord_only=False, keep_strand=False):
+ """Parse regions in any of the expected file formats.
+
+ Iterates over tuples of the tabular contents. Header lines are skipped.
+
+ Start and end coordinates are base-0, half-open.
+
+ If coord_only, yield triplets of (chrom, start, end). Otherwise, yield
+ quads of (chrom, start, end, name).
+ """
+ fmt = sniff_region_format(fname)
+ if fmt is None:
+ return []
+ if fmt == 'bed':
+ logging.info("Detected file format: BED")
+ elif fmt == 'interval':
+ logging.info("Detected file format: interval list")
+ parser = {'text': parse_text_coords,
+ 'interval': parse_interval_list,
+ 'bed': parse_bed,
+ }[fmt]
+ return parser(fname, coord_only, keep_strand)
+
+
+def report_bad_line(line_parser):
+ @functools.wraps(line_parser)
+ def wrapper(line):
+ try:
+ return line_parser(line)
+ except ValueError:
+ raise ValueError("Bad line: %r" % line)
+ return wrapper
+
+
+def parse_text_coords(fname, coord_only, _keep_strand):
+ """Parse text coordinates: chrom:start-end
+
+ Text coordinates are assumed to be counting from 1.
+ """
+ if coord_only:
+ @report_bad_line
+ def _parse_line(line):
+ chrom, _rest = line.rstrip().split(':', 1)
+ start, end = _rest.split('-')
+ if ':' in end:
+ end = end.split(':', 1)[0]
+ return chrom, int(start) - 1, int(end)
+ else:
+ @report_bad_line
+ def _parse_line(line):
+ fields = line.split(':')
+ if len(fields) == 3:
+ chrom, start_end, name = fields
+ elif len(fields) == 2:
+ chrom, start_end = fields
+ name = '-'
+ else:
+ raise ValueError
+ start, end = start_end.split('-')
+ return chrom, int(start) - 1, int(end), name.rstrip()
+
+ with as_handle(fname, 'rU') as handle:
+ for line in handle:
+ yield _parse_line(line)
+
+
+def parse_interval_list(fname, coord_only, keep_strand):
+ """Parse a Picard-compatible interval list.
+
+ Expected tabular columns:
+ chromosome, start position, end position, strand, region name
+
+ Counting is from 1.
+ """
+ if coord_only:
+ if keep_strand:
+ @report_bad_line
+ def _parse_line(line):
+ chrom, start, end, strand = line.split('\t')[:4]
+ return chrom, int(start) - 1, int(end), strand.rstrip()
+ else:
+ @report_bad_line
+ def _parse_line(line):
+ chrom, start, end = line.split('\t')[:3]
+ return chrom, int(start) - 1, int(end)
+ elif keep_strand:
+ @report_bad_line
+ def _parse_line(line):
+ fields = line.split('\t')
+ chrom, start, end, strand = fields[:4]
+ if len(fields) > 4:
+ name = fields[-1].rstrip()
+ else:
+ name = '-'
+ return chrom, int(start) - 1, int(end), name, strand
+ else:
+ @report_bad_line
+ def _parse_line(line):
+ fields = line.split('\t')
+ chrom, start, end = fields[:3]
+ if len(fields) > 3:
+ name = fields[-1].rstrip()
+ else:
+ name = '-'
+ return chrom, int(start) - 1, int(end), name
+
+ with as_handle(fname, 'rU') as handle:
+ for line in handle:
+ if line.startswith('@'):
+ # Skip the SAM header
+ continue
+ yield _parse_line(line)
+
+
+def parse_bed(fname, coord_only, keep_strand):
+ """Parse a BED file.
+
+ A BED file has these columns:
+ chromosome, start position, end position, [name, strand, other stuff...]
+
+ Counting is from 0.
+
+ Sets of regions are separated by "track" lines. This function stops
+ iteration after encountering a track line other than the first one in the
+ file.
+ """
+ if coord_only:
+ if keep_strand:
+ @report_bad_line
+ def _parse_line(line):
+ chrom, start, end, _name, _score, strand = line.split('\t', 6)[:6]
+ return chrom, int(start), int(end), strand.rstrip()
+ else:
+ @report_bad_line
+ def _parse_line(line):
+ chrom, start, end = line.split('\t', 3)[:3]
+ return chrom, int(start), int(end)
+ elif keep_strand:
+ @report_bad_line
+ def _parse_line(line):
+ fields = line.split('\t', 6)
+ chrom, start, end = fields[:3]
+ name = (fields[3].rstrip()
+ if len(fields) >= 4 else '-')
+ strand = (fields[5].rstrip()
+ if len(fields) >= 6 else '.')
+ return chrom, int(start), int(end), name, strand
+ else:
+ @report_bad_line
+ def _parse_line(line):
+ fields = line.split('\t', 4)
+ chrom, start, end = fields[:3]
+ name = (fields[3].rstrip()
+ if len(fields) >= 4 else '-')
+ return chrom, int(start), int(end), name
+
+ with as_handle(fname, 'rU') as handle:
+ firstline = next(handle)
+ if firstline.startswith("track"):
+ pass
+ else:
+ yield _parse_line(firstline)
+
+ for line in handle:
+ if line.startswith('track'):
+ raise StopIteration
+ yield _parse_line(line)
+
+
+def parse_bed_track(line):
+ """Parse the "name" field of a BED track definition line.
+
+ Example:
+ track name=146793_BastianLabv2_P2_target_region description="146793_BastianLabv2_P2_target_region"
+ """
+ fields = shlex.split(line) # raises ValueError if line is corrupted
+ assert fields[0] == 'track'
+ for field in fields[1:]:
+ if '=' in field:
+ key, val = field.split('=', 1)
+ if key == 'name':
+ return val
+ raise ValueError("No name defined for this track")
+
+
+def group_bed_tracks(bedfile):
+ """Group the parsed rows in a BED file by track.
+
+ Yields (track_name, iterable_of_lines), much like itertools.groupby.
+ """
+ # ENH - make this memory-efficient w/ generators or something
+ with as_handle(bedfile, 'r') as handle:
+ curr_track = 'DEFAULT'
+ curr_lines = []
+ for line in handle:
+ if line.startswith('track'):
+ if curr_lines:
+ yield curr_track, curr_lines
+ curr_lines = []
+ curr_track = parse_bed_track(line)
+ else:
+ curr_lines.append(line)
+ yield curr_track, curr_lines
+
diff --git a/cnvlib/ngfrills/samutil.py b/cnvlib/ngfrills/samutil.py
new file mode 100644
index 0000000..af1ee63
--- /dev/null
+++ b/cnvlib/ngfrills/samutil.py
@@ -0,0 +1,64 @@
+"""BAM utilities."""
+from __future__ import absolute_import, division, print_function
+
+import logging
+import os
+from itertools import islice
+
+import pysam
+
+from .shared import is_newer_than
+
+
+def ensure_bam_index(bam_fname):
+ """Ensure a BAM file is indexed, to enable fast traversal & lookup.
+
+ For MySample.bam, samtools will look for an index in these files, in order:
+
+ - MySample.bam.bai
+ - MySample.bai
+ """
+ if os.path.isfile(bam_fname + '.bai'):
+ # MySample.bam.bai
+ bai_fname = bam_fname + '.bai'
+ else:
+ # MySample.bai
+ bai_fname = bam_fname[:-1] + 'i'
+ if not is_newer_than(bai_fname, bam_fname):
+ logging.info("Indexing BAM file %s", bam_fname)
+ pysam.index(bam_fname)
+ bai_fname = bam_fname + '.bai'
+ assert os.path.isfile(bai_fname), \
+ "Failed to generate index " + bai_fname
+ return bai_fname
+
+
+def ensure_bam_sorted(bam_fname, by_name=False, span=50):
+ """Test if the reads in a BAM file are sorted as expected.
+
+ by_name=True: reads are expected to be sorted by query name. Consecutive
+ read IDs are in alphabetical order, and read pairs appear together.
+
+ by_name=False: reads are sorted by position. Consecutive reads have
+ increasing position.
+ """
+ if by_name:
+ # Compare read IDs
+ def out_of_order(read, prev):
+ return not (prev is None or
+ prev.qname <= read.qname)
+ else:
+ # Compare read locations
+ def out_of_order(read, prev):
+ return not (prev is None or
+ read.tid != prev.tid or
+ prev.pos <= read.pos)
+
+ # ENH - repeat at 50%, ~99% through the BAM
+ bam = pysam.Samfile(bam_fname, 'rb')
+ last_read = None
+ for read in islice(bam, span):
+ if out_of_order(read, last_read):
+ return False
+ last_read = read
+ return True
diff --git a/cnvlib/ngfrills/shared.py b/cnvlib/ngfrills/shared.py
new file mode 100644
index 0000000..bfde4bb
--- /dev/null
+++ b/cnvlib/ngfrills/shared.py
@@ -0,0 +1,10 @@
+"""NGS utilities."""
+from __future__ import absolute_import, division, print_function
+
+import os
+
+
+def is_newer_than(target_fname, orig_fname):
+ if not os.path.isfile(target_fname):
+ return False
+ return (os.stat(target_fname).st_mtime >= os.stat(orig_fname).st_mtime)
diff --git a/cnvlib/parallel.py b/cnvlib/parallel.py
new file mode 100644
index 0000000..7a555d0
--- /dev/null
+++ b/cnvlib/parallel.py
@@ -0,0 +1,26 @@
+"""Utilities for multi-core parallel processing."""
+from __future__ import absolute_import, division, print_function
+import multiprocessing
+
+
+class SerialPool(object):
+ """Mimic the multiprocessing.Pool interface, but run in serial."""
+
+ def __init__(self):
+ pass
+
+ def apply_async(self, func, args):
+ """Just call the function."""
+ func(*args)
+
+ # No-ops to mimic multiprocessing.Pool
+ def close(self): pass
+ def join(self): pass
+
+
+def pick_pool(nprocs):
+ if nprocs == 1:
+ return SerialPool()
+ if nprocs < 1:
+ nprocs = multiprocessing.cpu_count()
+ return multiprocessing.Pool(nprocs)
diff --git a/cnvlib/params.py b/cnvlib/params.py
new file mode 100644
index 0000000..8f25aa1
--- /dev/null
+++ b/cnvlib/params.py
@@ -0,0 +1,15 @@
+"""Hard-coded parameters for CNVkit. These should not change between runs."""
+# Filters used in constructing the reference
+MIN_REF_COVERAGE = -5.0
+MAX_REF_SPREAD = 1.0
+NULL_LOG2_COVERAGE = -20.0
+
+# Sequencing read length (or average, if it varies)
+READ_LEN = 100
+
+# Fragment size for paired-end reads
+INSERT_SIZE = 250
+
+# Target/bin names that are not meaningful gene names
+# (In some UCSF panels, "CGH" probes denote selected intergenic regions)
+IGNORE_GENE_NAMES = ("-", ".", "CGH")
diff --git a/cnvlib/plots.py b/cnvlib/plots.py
new file mode 100644
index 0000000..769ac7e
--- /dev/null
+++ b/cnvlib/plots.py
@@ -0,0 +1,572 @@
+"""Plotting utilities."""
+from __future__ import absolute_import, division
+
+import collections
+import logging
+import math
+import sys
+
+import numpy as np
+
+from Bio._py3k import zip
+iteritems = (dict.iteritems if sys.version_info[0] < 3 else dict.items)
+
+from . import core, params, smoothing
+
+SEG_COLOR = "darkorange"
+POINT_COLOR = '#606060'
+
+# === Chromosome-level scatter plots ===
+
+HIGHLIGHT_COLOR = 'gold'
+MB = 1e-6 # To rescale from bases to megabases
+
+def setup_chromosome(axis, probes=None, segments=None, variants=None,
+ y_min=None, y_max=None, y_label=None):
+ """Configure axes for plotting a single chromosome's data.
+
+ `probes`, `segments`, and `variants` should already be subsetted to the
+ region that will be plotted.
+ """
+ min_x = np.inf
+ max_x = 0
+ for arr in (probes, segments, variants):
+ if arr:
+ max_x = max(max_x, arr.end.iat[-1])
+ min_x = min(min_x, arr.start.iat[0])
+ if not max_x > min_x:
+ if any((probes, segments, variants)):
+ logging.warn("*WARNING* selection start %s > end %s (%s); did you "
+ "correctly sort the input file by genomic location?",
+ min_x, max_x, max_x - min_x)
+ raise ValueError("No usable data points to plot out of "
+ "%d probes, %d segments, %d variants"
+ % (len(probes) if probes else 0,
+ len(segments) if segments else 0,
+ len(variants) if variants else 0))
+ axis.set_xlim(min_x * MB, max_x * MB)
+ if y_min and y_max:
+ axis.set_ylim(y_min, y_max)
+ if y_min < 0 < y_max:
+ axis.axhline(color='k')
+ if y_label:
+ axis.set_ylabel(y_label)
+ axis.tick_params(which='both', direction='out')
+ axis.get_xaxis().tick_bottom()
+ axis.get_yaxis().tick_left()
+
+
+def cnv_on_chromosome(axis, probes, segments, genes, background_marker=None,
+ do_trend=False, y_min=None, y_max=None):
+ """Draw a scatter plot of probe values with CBS calls overlaid.
+
+ Argument 'genes' is a list of tuples: (start, end, gene name)
+ """
+ # Get scatter plot coordinates
+ x = 0.5 * (probes['start'] + probes['end']) * MB # bin midpoints
+ y = probes['log2']
+ if 'weight' in probes:
+ w = 46 * probes['weight'] ** 2 + 2
+ else:
+ w = np.repeat(30, len(x))
+ is_bg = (probes['gene'] == 'Background')
+
+ # Configure axes
+ # TODO - use segment y-values if probes not given
+ if not y_min:
+ y_min = max(-5.0, min(y.min() - .1, -.3)) if len(y) else -1.1
+ if not y_max:
+ y_max = max(.3, y.max() + (.25 if genes else .1)) if len(y) else 1.1
+ setup_chromosome(axis, probes, segments, None, y_min, y_max,
+ "Copy ratio (log2)")
+ if genes:
+ # Rotate text in proportion to gene density
+ ngenes = len(genes)
+ text_size = ('small' if ngenes <= 6 else 'x-small')
+ if ngenes <= 3:
+ text_rot = 'horizontal'
+ elif ngenes <= 6:
+ text_rot = 30
+ elif ngenes <= 10:
+ text_rot = 45
+ elif ngenes <= 20:
+ text_rot = 60
+ else:
+ text_rot = 'vertical'
+ for gene in genes:
+ gene_start, gene_end, gene_name = gene
+ # Highlight and label gene region
+ # (rescale positions from bases to megabases)
+ axis.axvspan(gene_start * MB, gene_end * MB,
+ alpha=0.5, color=HIGHLIGHT_COLOR, zorder=-1)
+ axis.text(0.5 * (gene_start + gene_end) * MB,
+ min(2.4, y.max() + .1) if len(y) else .1,
+ gene_name,
+ horizontalalignment='center',
+ rotation=text_rot,
+ size=text_size)
+ # size='small')
+
+ if background_marker in (None, 'o'):
+ # Plot targets and antitargets with the same marker
+ axis.scatter(x, y, w, color=POINT_COLOR, alpha=0.4, marker='o')
+ else:
+ # Use the given marker to plot antitargets separately
+ x_fg = []
+ y_fg = []
+ w_fg = []
+ x_bg = []
+ y_bg = []
+ # w_bg = []
+ for x_pt, y_pt, w_pt, is_bg_pt in zip(x, y, w, is_bg):
+ if is_bg_pt:
+ x_bg.append(x_pt)
+ y_bg.append(y_pt)
+ # w_bg.append(w_pt)
+ else:
+ x_fg.append(x_pt)
+ y_fg.append(y_pt)
+ w_fg.append(w_pt)
+ axis.scatter(x_fg, y_fg, w_fg, color=POINT_COLOR, alpha=0.4, marker='o')
+ axis.scatter(x_bg, y_bg, color=POINT_COLOR, alpha=0.5,
+ marker=background_marker)
+
+ # Add a local trend line
+ if do_trend:
+ axis.plot(x, smoothing.smoothed(y, 50),
+ color=POINT_COLOR, linewidth=2, zorder=-1)
+
+ # Get coordinates for CBS lines & draw them
+ if segments:
+ for row in segments:
+ axis.plot((row.start * MB, row.end * MB),
+ (row.log2, row.log2),
+ color=SEG_COLOR, linewidth=4, solid_capstyle='round')
+
+
+def snv_on_chromosome(axis, variants, segments, genes,
+ do_trend, do_boost=False):
+ # XXX only set x-limits if not already done for probes/segments
+ # setup_chromosome(axis, None, segments, variants,
+ # 0.0, 1.0, "VAF")
+ axis.set_ylim(0.0, 1.0)
+ axis.set_ylabel("VAF")
+ axis.set_xlabel("Position (Mb)")
+ axis.get_yaxis().tick_left()
+ axis.get_xaxis().tick_top()
+ axis.tick_params(which='both', direction='out',
+ labelbottom=False, labeltop=False)
+
+ x_mb = variants["start"] * MB
+ if do_boost:
+ y = variants.tumor_boost()
+ else:
+ y = np.asfarray(variants["alt_freq"])
+ axis.scatter(x_mb, y, color=POINT_COLOR, alpha=0.3)
+ # TODO - highlight genes/selection
+ if segments:
+ # Draw average VAF within each segment
+ posns = np.asfarray(variants["start"]) # * MB
+ y = np.asfarray(y)
+ for v_start, v_end, v_freq in group_snvs_by_segments(posns, y,
+ segments):
+ # ENH: color by segment gain/loss
+ axis.plot([v_start * MB, v_end * MB], [v_freq, v_freq],
+ color='#C0C0C0', linewidth=2, #zorder=1,
+ solid_capstyle='round')
+
+# === Genome-level scatter plots ===
+
+def setup_genome(axis, probes, segments, variants, y_min=None, y_max=None):
+ """Configure axes for plotting a whole genomes's data."""
+ pass
+
+
+def cnv_on_genome(axis, probes, segments, pad, do_trend=False, y_min=None,
+ y_max=None):
+ """Plot coverages and CBS calls for all chromosomes on one plot."""
+ # Group probes by chromosome (to calculate plotting coordinates)
+ if probes:
+ chrom_probe_centers = {chrom: 0.5 * (rows['start'] + rows['end'])
+ for chrom, rows in probes.by_chromosome()}
+ chrom_sizes = chromosome_sizes(probes)
+ else:
+ chrom_sizes = chromosome_sizes(segments)
+
+ # Same for segment calls
+ chrom_seg_coords = {chrom: zip(rows['log2'], rows['start'], rows['end'])
+ for chrom, rows in segments.by_chromosome()
+ } if segments else {}
+
+ x_starts = plot_x_dividers(axis, chrom_sizes, pad)
+ x = []
+ seg_lines = [] # y-val, x-start, x-end
+ for chrom, curr_offset in x_starts.items():
+ if probes:
+ x.extend(chrom_probe_centers[chrom] + curr_offset)
+ if chrom in chrom_seg_coords:
+ seg_lines.extend((c[0], c[1] + curr_offset, c[2] + curr_offset)
+ for c in chrom_seg_coords[chrom])
+
+ # Configure axes etc.
+ axis.axhline(color='k')
+ axis.set_ylabel("Copy ratio (log2)")
+ if not (y_min and y_max):
+ if segments:
+ # Auto-scale y-axis according to segment mean-coverage values
+ seg_auto_vals = segments[(segments.chromosome != 'chr6') &
+ (segments.chromosome != 'chrY')]['log2']
+ if not y_min:
+ y_min = min(seg_auto_vals.min() - .2, -1.5)
+ if not y_max:
+ y_max = max(seg_auto_vals.max() + .2, 1.5)
+ else:
+ if not y_min:
+ y_min = -2.5
+ if not y_max:
+ y_max = 2.5
+ axis.set_ylim(y_min, y_max)
+
+ # Plot points
+ if probes:
+ axis.scatter(x, probes['log2'], color=POINT_COLOR, edgecolor='none',
+ alpha=0.2, marker='.')
+ # Add a local trend line
+ if do_trend:
+ axis.plot(x, _smooth_genome_log2(probes, smoothing.smoothed, 150),
+ color=POINT_COLOR, linewidth=2, zorder=-1)
+ # Plot segments
+ for seg_line in seg_lines:
+ y1, x1, x2 = seg_line
+ axis.plot((x1, x2), (y1, y1),
+ color=SEG_COLOR, linewidth=3, solid_capstyle='round')
+
+
+def _smooth_genome_log2(cnarr, smooth_func, width):
+ """Fit a trendline through bin log2 ratios, handling chromosome boundaries.
+
+ Returns an array of smoothed log2 values, calculated with `smooth_func`
+ and `width`, equal in length to `cnarr`.
+ """
+ # ENH: also split by centromeres (long internal gaps -- see PSCBS)
+ # ENH: use pandas groupby
+ out = {chrom: smooth_func(subcna['log2'], width)
+ for chrom, subcna in cnarr.by_chromosome()}
+ return np.concatenate(
+ [out[chrom] for chrom in sorted(out, key=core.sorter_chrom)])
+
+
+def snv_on_genome(axis, variants, chrom_sizes, segments, do_trend, pad,
+ do_boost=False):
+ """Plot a scatter-plot of SNP chromosomal positions and shifts."""
+ axis.set_ylim(0.0, 1.0)
+ axis.set_ylabel("VAF")
+ x_starts = plot_x_dividers(axis, chrom_sizes, pad)
+
+ # Calculate the coordinates of plot components
+ chrom_snvs = dict(variants.by_chromosome())
+ x_posns_chrom = {}
+ y_posns_chrom = {}
+ trends = []
+ for chrom, curr_offset in iteritems(x_starts):
+ snvs = chrom_snvs.get(chrom, [])
+ if not len(snvs):
+ x_posns_chrom[chrom] = []
+ y_posns_chrom[chrom] = []
+ continue
+ posns = np.asfarray(snvs["start"])
+ x_posns = posns + curr_offset
+ if do_boost:
+ vafs = snvs.tumor_boost()
+ else:
+ vafs = np.asfarray(snvs["alt_freq"])
+ x_posns_chrom[chrom] = x_posns
+ y_posns_chrom[chrom] = vafs
+ # Trend bars: always calculated, only shown on request
+ if segments:
+ # Draw average VAF within each segment
+ for v_start, v_end, v_freq in group_snvs_by_segments(posns, vafs,
+ segments,
+ chrom):
+ trends.append((v_start + curr_offset, v_end + curr_offset,
+ v_freq))
+ else:
+ # Draw chromosome-wide average VAF
+ for mask_vaf in ((vafs > 0.5), (vafs <= 0.5)):
+ if sum(mask_vaf) > 1:
+ these_posns = x_posns[mask_vaf]
+ trends.append((these_posns[0], these_posns[-1],
+ np.median(vafs[mask_vaf])))
+
+ # Test for significant shifts in VAF
+ # ENH - use segments if provided
+ # if significant, colorize those points / that median line
+ sig_chroms = [] # test_loh(partition_by_chrom(chrom_snvs))
+
+ # Render significantly shifted heterozygous regions separately
+ x_posns = []
+ y_posns = []
+ x_posns_sig = []
+ y_posns_sig = []
+ for chrom in chrom_sizes:
+ posns = x_posns_chrom[chrom]
+ vafs = y_posns_chrom[chrom]
+ if chrom in sig_chroms:
+ x_posns_sig.extend(posns)
+ y_posns_sig.extend(vafs)
+ else:
+ x_posns.extend(posns)
+ y_posns.extend(vafs)
+
+ # Plot the points
+ axis.scatter(x_posns, y_posns, color=POINT_COLOR, edgecolor='none',
+ alpha=0.2, marker='.')
+ axis.scatter(x_posns_sig, y_posns_sig, color='salmon', edgecolor='none',
+ alpha=0.3)
+ # Add trend lines to each chromosome
+ if do_trend or segments:
+ # Draw a line across each chromosome at the median shift level
+ for x_start, x_end, y_trend in trends:
+ # ENH: color by segment gain/loss
+ axis.plot([x_start, x_end], [y_trend, y_trend],
+ color='#C0C0C0', linewidth=2, zorder=-1,
+ solid_capstyle='round')
+
+
+
+# XXX use by_ranges
+def group_snvs_by_segments(snv_posns, snv_freqs, segments, chrom=None):
+ """Group SNP allele frequencies by segment.
+
+ Return an iterable of: start, end, value.
+ """
+ if chrom:
+ segments = segments.select(chromosome=chrom)
+ seg_starts = segments.start
+ # Assign a segment number to each variant, basically
+ indices = np.maximum(seg_starts.searchsorted(snv_posns), 1) - 1
+ for i in sorted(set(indices)):
+ mask_in_seg = (indices == i)
+ freqs = snv_freqs[mask_in_seg]
+ posns = snv_posns[mask_in_seg]
+ # Separately emit VAFs above and below .5 for plotting
+ mask_above_mid = (freqs > 0.5)
+ for mask_vaf in (mask_above_mid, ~mask_above_mid):
+ if sum(mask_vaf) > 1:
+ these_posns = posns[mask_vaf]
+ yield (these_posns[0], these_posns[-1],
+ np.median(freqs[mask_vaf]))
+
+
+def plot_x_dividers(axis, chrom_sizes, pad):
+ """Plot vertical dividers and x-axis labels given the chromosome sizes.
+
+ Returns a table of the x-position offsets of each chromosome.
+
+ Draws vertical black lines between each chromosome, with padding.
+ Labels each chromosome range with the chromosome name, centered in the
+ region, under a tick.
+ Sets the x-axis limits to the covered range.
+ """
+ assert isinstance(chrom_sizes, collections.OrderedDict)
+
+ x_dividers = []
+ x_centers = []
+ x_starts = collections.OrderedDict()
+ curr_offset = pad
+ for label, size in chrom_sizes.items():
+ x_starts[label] = curr_offset
+ x_centers.append(curr_offset + 0.5 * size)
+ x_dividers.append(curr_offset + size + pad)
+ curr_offset += size + 2 * pad
+
+ axis.set_xlim(0, curr_offset)
+ for xposn in x_dividers[:-1]:
+ axis.axvline(x=xposn, color='k')
+ # Use chromosome names as x-axis labels (instead of base positions)
+ axis.set_xticks(x_centers)
+ axis.set_xticklabels(chrom_sizes.keys(), rotation=60)
+ axis.tick_params(labelsize='small')
+ axis.tick_params(axis='x', length=0)
+ axis.get_yaxis().tick_left()
+
+ return x_starts
+
+
+# ________________________________________
+# Internal supporting functions
+
+def chromosome_sizes(probes, to_mb=False):
+ """Create an ordered mapping of chromosome names to sizes."""
+ chrom_sizes = collections.OrderedDict()
+ for chrom, rows in probes.by_chromosome():
+ chrom_sizes[chrom] = rows['end'].max()
+ if to_mb:
+ chrom_sizes[chrom] *= MB
+ return chrom_sizes
+
+
+def partition_by_chrom(chrom_snvs):
+ """Group the tumor shift values by chromosome (for statistical testing)."""
+ chromnames = set(chrom_snvs.keys())
+ bins = {key: {'thisbin': [], 'otherbins': []}
+ for key in chrom_snvs}
+ for thischrom, snvs in iteritems(chrom_snvs):
+ shiftvals = np.array([abs(v[2]) for v in snvs])
+ bins[thischrom]['thisbin'].extend(shiftvals)
+ for otherchrom in chromnames:
+ if otherchrom == thischrom:
+ continue
+ bins[otherchrom]['otherbins'].extend(shiftvals)
+ return bins
+
+
+def test_loh(bins, alpha=0.0025):
+ """Test each chromosome's SNP shifts and the combined others'.
+
+ The statistical test is Mann-Whitney, a one-sided non-parametric test for
+ difference in means.
+ """
+ # TODO - this doesn't work right if there are many shifted regions
+ try:
+ from scipy import stats
+ except ImportError:
+ # SciPy not installed; can't test for significance
+ return []
+
+ significant_chroms = []
+ for chrom, partitions in iteritems(bins):
+ these_shifts = np.array(partitions['thisbin'], np.float_)
+ other_shifts = np.array(partitions['otherbins'], np.float_)
+ if len(these_shifts) < 20:
+ logging.info("Too few points (%d) to test chrom %s",
+ len(these_shifts), chrom)
+ elif these_shifts.mean() > other_shifts.mean():
+ logging.debug("\nThese ~= %f (N=%d), Other ~= %f (N=%d)",
+ these_shifts.mean(), len(these_shifts),
+ other_shifts.mean(), len(other_shifts))
+ u, prob = stats.mannwhitneyu(these_shifts, other_shifts)
+ logging.info("Mann-Whitney - %s: u=%s, p=%s", chrom, u, prob)
+ if prob < alpha:
+ significant_chroms.append(chrom)
+
+ return significant_chroms
+
+
+# ________________________________________
+# Utilies used by other modules
+
+
+def cvg2rgb(cvg, desaturate):
+ """Choose a shade of red or blue representing log2-coverage value."""
+ cutoff = 1.33 # Values above this magnitude are shown with max intensity
+ x = min(abs(cvg) / cutoff, 1.0)
+ if desaturate:
+ # Adjust intensity sigmoidally -- reduce near 0, boost near 1
+ # Exponent <1 shifts the fixed point leftward (from x=0.5)
+ x = ((1. - math.cos(x * math.pi)) / 2.) ** 0.8
+ # Slight desaturation of colors at lower coverage
+ s = x**1.2
+ else:
+ s = x
+ if cvg < 0:
+ rgb = (1 - s, 1 - s, 1 - .25*x) # Blueish
+ else:
+ rgb = (1 - .25*x, 1 - s, 1 - s) # Reddish
+ return rgb
+
+
+# XXX should this be a CopyNumArray method?
+# or: use by_genes internally
+# or: have by_genes use this internally
+def gene_coords_by_name(probes, names):
+ """Find the chromosomal position of each named gene in probes.
+
+ Returns a dict: {chromosome: [(start, end, gene name), ...]}
+ """
+ # Create an index of gene names
+ gene_index = collections.defaultdict(set)
+ for i, gene in enumerate(probes['gene']):
+ for gene_name in gene.split(','):
+ if gene_name in names:
+ gene_index[gene_name].add(i)
+ # Retrieve coordinates by name
+ all_coords = collections.defaultdict(lambda : collections.defaultdict(set))
+ for name in names:
+ gene_probes = probes.data.take(sorted(gene_index.get(name, [])))
+ if not len(gene_probes):
+ raise ValueError("No targeted gene named '%s' found" % name)
+ # Find the genomic range of this gene's probes
+ start = gene_probes['start'].min()
+ end = gene_probes['end'].max()
+ chrom = core.check_unique(gene_probes['chromosome'], name)
+ # Deduce the unique set of gene names for this region
+ orig_names = set()
+ for oname in set(gene_probes['gene']):
+ orig_names.update(oname.split(','))
+ all_coords[chrom][start, end].update(orig_names)
+ # Consolidate each region's gene names into a string
+ uniq_coords = {}
+ for chrom, hits in all_coords.iteritems():
+ uniq_coords[chrom] = [(start, end, ",".join(sorted(orig_names)))
+ for (start, end), orig_names in hits.iteritems()]
+ return uniq_coords
+
+
+def gene_coords_by_range(probes, chrom, start, end,
+ ignore=params.IGNORE_GENE_NAMES):
+ """Find the chromosomal position of all genes in a range.
+
+ Returns a dict: {chromosome: [(start, end, gene), ...]}
+ """
+ ignore += ('Background',)
+ # Tabulate the genes in the selected region
+ genes = collections.OrderedDict()
+ for row in probes.in_range(chrom, start, end):
+ name = str(row.gene)
+ if name in ignore:
+ continue
+ if name in genes:
+ genes[name][1] = row.end
+ else:
+ genes[name] = [row.start, row.end]
+ # Reorganize the data structure
+ return {chrom: [(gstart, gend, name)
+ for name, (gstart, gend) in genes.items()]}
+
+
+def unpack_range(a_range):
+ """Extract chromosome, start, end from a string or tuple.
+
+ Examples:
+
+ "chr1" -> ("chr1", None, None)
+ "chr1:100-123" -> ("chr1", 100, 123)
+ ("chr1", 100, 123) -> ("chr1", 100, 123)
+ """
+ if not a_range:
+ return None, None, None
+ if isinstance(a_range, basestring):
+ if ':' in a_range or '-' in a_range:
+ return parse_range_text(a_range)
+ return a_range, None, None
+ if isinstance(a_range, (list, tuple)) and len(a_range) == 3:
+ return tuple(a_range)
+ raise ValueError("Not a range: %r" % a_range)
+
+
+def parse_range_text(text):
+ """Parse a chromosomal range specification.
+
+ Range spec string should look like ``chr1:1234-5678`` or ``chr1:1234-`` or
+ ``chr1:-5678``, where missing start becomes 0 and missing end becomes None.
+ """
+ try:
+ chrom, rest = text.split(':')
+ start, end = rest.split('-')
+ start = int(start) if start else 0
+ end = int(end) if end else None
+ return chrom, start, end
+ except Exception:
+ raise ValueError("Invalid range spec: " + text
+ + " (should be like: chr1:2333000-2444000)")
diff --git a/cnvlib/rary.py b/cnvlib/rary.py
new file mode 100644
index 0000000..946a8f7
--- /dev/null
+++ b/cnvlib/rary.py
@@ -0,0 +1,176 @@
+"""An array of genomic regions or features."""
+from __future__ import absolute_import, division, print_function
+
+import logging
+import sys
+
+import pandas as pd
+from Bio.File import as_handle
+
+from . import core, gary, ngfrills
+
+
+class RegionArray(gary.GenomicArray):
+ """An array of genomic intervals."""
+ _required_columns = ("chromosome", "start", "end",
+ # "name", "strand",
+ )
+ _required_dtypes = ("string", "int", "int")
+
+ def __init__(self, data_table, meta_dict=None):
+ gary.GenomicArray.__init__(self, data_table, meta_dict)
+
+ @classmethod
+ def read(cls, fname, sample_id=None, fmt=None):
+ """Read regions in any of the expected file formats.
+
+ Iterates over tuples of the tabular contents. Header lines are skipped.
+
+ Start and end coordinates are base-0, half-open.
+ """
+ if sample_id is None:
+ if isinstance(fname, basestring):
+ sample_id = core.fbase(fname)
+ elif fmt is None:
+ raise ValueError("To read regions from a stream, the file "
+ "format must be specified with the `fmt` "
+ "argument.")
+ else:
+ sample_id = '<unknown>'
+
+ if not fmt:
+ fmt = ngfrills.sniff_region_format(fname)
+ if fmt is None:
+ return cls([])
+ if fmt == 'bed':
+ logging.info("Detected file format: BED")
+ elif fmt == 'interval':
+ logging.info("Detected file format: interval list")
+ parser = {'text': _parse_text_coords,
+ 'interval': _parse_interval_list,
+ 'bed': _parse_bed,
+ }[fmt]
+ table = parser(fname)
+ return cls(table, {"sample_id": sample_id})
+
+ def write(self, outfile=sys.stdout, fmt="bed", verbose=True):
+ assert fmt in ("text", "interval") or fmt.startswith("bed")
+ if fmt == "text":
+ cp = self.copy()
+ cp['start'] += 1
+ table = cp.labels()
+ else:
+ table = self.data
+ if fmt == "interval":
+ table["start"] += 1
+ if "name" not in table:
+ table["name"] = '-'
+ if "strand" not in table:
+ table["strand"] = "+"
+ table = table.loc[:, ["chromosome", "start", "end", "strand",
+ "name"]]
+ elif fmt == "bed4":
+ if "name" not in table:
+ table["name"] = '-'
+ table = table.loc[:, ["chromosome", "start", "end", "name"]]
+ elif fmt == "bed3":
+ table = table.loc[:, ["chromosome", "start", "end"]]
+ # Default: bed-like, keep all trailing columns
+
+ with ngfrills.safe_write(outfile, False) as outfile:
+ table.to_csv(outfile, sep='\t', header=False, index=False)
+ if verbose:
+ # Log the output path, if possible
+ if isinstance(outfile, basestring):
+ outfname = outfile
+ elif hasattr(outfile, 'name') and outfile not in (sys.stdout,
+ sys.stderr):
+ outfname = outfile.name
+ else:
+ # Probably stdout or stderr used in a pipeline -- don't pollute
+ return
+ logging.info("Wrote %s with %d regions", outfname, len(table))
+
+
+
+def _parse_text_coords(infile):
+ """Parse text coordinates: chrom:start-end
+
+ Or sometimes: chrom:start-end:name
+
+ Text coordinates are assumed to be counting from 1.
+ """
+ @ngfrills.report_bad_line
+ def _parse_line(line):
+ fields = line.split(':')
+ if len(fields) == 3:
+ chrom, start_end, name = fields
+ elif len(fields) == 2:
+ chrom, start_end = fields
+ name = '-'
+ else:
+ raise ValueError("Bad line: %r" % line)
+ start, end = start_end.split('-')
+ return chrom, int(start) - 1, int(end), name.rstrip()
+
+ with as_handle(infile, 'rU') as handle:
+ rows = [_parse_line(line) for line in handle]
+ return pd.DataFrame.from_records(rows, columns=["chromosome", "start",
+ "end", "name"])
+
+
+def _parse_interval_list(infile):
+ """Parse a Picard-compatible interval list.
+
+ Expected tabular columns:
+ chromosome, start position, end position, strand, region name
+
+ Counting is from 1.
+ """
+ table = pd.read_table(infile,
+ comment='@', # Skip the SAM header
+ names=["chromosome", "start", "end", "strand", "name",
+ ])
+ table["name"].fillna('-', inplace=True)
+ table["start"] -= 1
+ return table
+
+
+def _parse_bed(infile):
+ """Parse a BED file.
+
+ A BED file has these columns:
+ chromosome, start position, end position, [name, strand, other stuff...]
+
+ Counting is from 0.
+
+ Sets of regions are separated by "track" lines. This function stops reading
+ after encountering a track line other than the first one in the file.
+ """
+ # ENH: just pd.read_table, skip 'track'
+ @ngfrills.report_bad_line
+ def _parse_line(line):
+ fields = line.split('\t', 6)
+ chrom, start, end = fields[:3]
+ name = (fields[3].rstrip()
+ if len(fields) >= 4 else '-')
+ strand = (fields[5].rstrip()
+ if len(fields) >= 6 else '.')
+ return chrom, int(start), int(end), name, strand
+
+ def track2track(handle):
+ firstline = next(handle)
+ if firstline.startswith("track"):
+ pass
+ else:
+ yield firstline
+ for line in handle:
+ if line.startswith('track'):
+ raise StopIteration
+ yield line
+
+ with as_handle(infile, 'rU') as handle:
+ rows = map(_parse_line, track2track(handle))
+ return pd.DataFrame.from_records(rows, columns=["chromosome", "start",
+ "end", "name", "strand"])
+
diff --git a/cnvlib/reference.py b/cnvlib/reference.py
new file mode 100644
index 0000000..63cb5df
--- /dev/null
+++ b/cnvlib/reference.py
@@ -0,0 +1,240 @@
+"""Supporting functions for the 'reference' command."""
+from __future__ import absolute_import, division, print_function
+
+import logging
+
+import numpy as np
+from Bio._py3k import map, zip
+
+from . import core, fix, metrics, ngfrills, params
+from .cnary import CopyNumArray as CNA
+from .rary import RegionArray as RA
+
+
+def bed2probes(bed_fname):
+ """Create neutral-coverage probes from intervals."""
+ regions = RA.read(bed_fname)
+ table = regions.data.loc[:, ("chromosome", "start", "end")]
+ table["gene"] = (regions.data["name"] if "name" in regions.data else '-')
+ table["log2"] = 0.0
+ table["spread"] = 0.0
+ return CNA(table, {"sample_id": core.fbase(bed_fname)})
+
+
+def combine_probes(filenames, fa_fname, is_male_reference, skip_low,
+ fix_gc, fix_edge, fix_rmask):
+ """Calculate the median coverage of each bin across multiple samples.
+
+ Input:
+ List of .cnn files, as generated by 'coverage' or 'import-picard'.
+ `fa_fname`: fil columns for GC and RepeatMasker genomic values.
+ Returns:
+ A single CopyNumArray summarizing the coverages of the input samples,
+ including each bin's "average" coverage, "spread" of coverages, and
+ genomic GC content.
+ """
+ columns = {}
+
+ # Load coverage from target/antitarget files
+ logging.info("Loading %s", filenames[0])
+ cnarr1 = CNA.read(filenames[0])
+ if not len(cnarr1):
+ # Just create an empty array with the right columns
+ col_names = ['chromosome', 'start', 'end', 'gene', 'log2']
+ if 'gc' in cnarr1 or fa_fname:
+ col_names.append('gc')
+ if fa_fname:
+ col_names.append('rmask')
+ col_names.append('spread')
+ return CNA.from_rows([], col_names, {'sample_id': "reference"})
+
+ # Calculate GC and RepeatMasker content for each probe's genomic region
+ if fa_fname and (fix_rmask or fix_gc):
+ gc, rmask = get_fasta_stats(cnarr1, fa_fname)
+ if fix_gc:
+ columns['gc'] = gc
+ if fix_rmask:
+ columns['rmask'] = rmask
+ elif 'gc' in cnarr1 and fix_gc:
+ # Reuse .cnn GC values if they're already stored (via import-picard)
+ gc = cnarr1['gc']
+ columns['gc'] = gc
+
+ # Make the sex-chromosome coverages of male and female samples compatible
+ is_chr_x = (cnarr1.chromosome == cnarr1._chr_x_label)
+ is_chr_y = (cnarr1.chromosome == cnarr1._chr_y_label)
+ flat_coverage = cnarr1.expect_flat_cvg(is_male_reference)
+ def shift_sex_chroms(cnarr):
+ """Shift sample X and Y chromosomes to match the reference gender.
+
+ Reference values:
+ XY: chrX -1, chrY -1
+ XX: chrX 0, chrY -1
+
+ Plan:
+ chrX:
+ xx sample, xx ref: 0 (from 0)
+ xx sample, xy ref: -= 1 (from -1)
+ xy sample, xx ref: += 1 (from 0) +1
+ xy sample, xy ref: 0 (from -1) +1
+ chrY:
+ xx sample, xx ref: = -1 (from -1)
+ xx sample, xy ref: = -1 (from -1)
+ xy sample, xx ref: 0 (from -1) +1
+ xy sample, xy ref: 0 (from -1) +1
+
+ """
+ is_sample_female = cnarr.guess_xx()
+ cnarr['log2'] += flat_coverage
+ if is_sample_female:
+ # chrX already OK
+ # No chrY; it's all noise, so just match the male
+ cnarr[is_chr_y, 'log2'] = -1.0
+ else:
+ # 1/2 #copies of each sex chromosome
+ cnarr[is_chr_x | is_chr_y, 'log2'] += 1.0
+
+ edge_bias = fix.get_edge_bias(cnarr1, params.INSERT_SIZE)
+ def bias_correct_coverage(cnarr):
+ """Perform bias corrections on the sample."""
+ cnarr.center_all(skip_low=skip_low)
+ shift_sex_chroms(cnarr)
+ # Skip bias corrections if most bins have no coverage (e.g. user error)
+ if (cnarr['log2'] > params.NULL_LOG2_COVERAGE - params.MIN_REF_COVERAGE
+ ).sum() <= len(cnarr) // 2:
+ logging.warn("WARNING: most bins have no or very low coverage; "
+ "check that the right BED file was used")
+ else:
+ if 'gc' in columns and fix_gc:
+ logging.info("Correcting for GC bias...")
+ cnarr = fix.center_by_window(cnarr, .1, columns['gc'])
+ if 'rmask' in columns and fix_rmask:
+ logging.info("Correcting for RepeatMasker bias...")
+ cnarr = fix.center_by_window(cnarr, .1, columns['rmask'])
+ if fix_edge:
+ logging.info("Correcting for density bias...")
+ cnarr = fix.center_by_window(cnarr, .1, edge_bias)
+ return cnarr['log2']
+
+ # Pseudocount of 1 "flat" sample
+ all_coverages = [flat_coverage, bias_correct_coverage(cnarr1)]
+ for fname in filenames[1:]:
+ logging.info("Loading target %s", fname)
+ cnarrx = CNA.read(fname)
+ # Bin information should match across all files
+ if not (len(cnarr1) == len(cnarrx)
+ and (cnarr1.chromosome == cnarrx.chromosome).all()
+ and (cnarr1.start == cnarrx.start).all()
+ and (cnarr1.end == cnarrx.end).all()
+ and (cnarr1['gene'] == cnarrx['gene']).all()):
+ raise RuntimeError("%s probes do not match those in %s"
+ % (fname, filenames[0]))
+ all_coverages.append(bias_correct_coverage(cnarrx))
+ all_coverages = np.vstack(all_coverages)
+
+ logging.info("Calculating average bin coverages")
+ cvg_centers = np.apply_along_axis(metrics.biweight_location, 0,
+ all_coverages)
+ logging.info("Calculating bin spreads")
+ spreads = np.apply_along_axis(metrics.biweight_midvariance, 0,
+ all_coverages)
+ columns['spread'] = spreads
+ columns.update({
+ 'chromosome': cnarr1.chromosome,
+ 'start': cnarr1.start,
+ 'end': cnarr1.end,
+ 'gene': cnarr1['gene'],
+ 'log2': cvg_centers,
+ })
+ return CNA.from_columns(columns, {'sample_id': "reference"})
+
+
+def warn_bad_probes(probes):
+ """Warn about target probes where coverage is poor.
+
+ Prints a formatted table to stderr.
+ """
+ bad_probes = probes[fix.mask_bad_probes(probes)]
+ fg_index = (bad_probes['gene'] != 'Background')
+ fg_bad_probes = bad_probes[fg_index]
+ if len(fg_bad_probes) > 0:
+ bad_pct = 100 * len(fg_bad_probes) / sum(probes['gene'] != 'Background')
+ logging.info("Targets: %d (%s) bins failed filters:",
+ len(fg_bad_probes), "%.4f" % bad_pct + '%')
+ gene_cols = max(map(len, fg_bad_probes['gene']))
+ labels = list(map(CNA.row2label, fg_bad_probes))
+ chrom_cols = max(map(len, labels))
+ last_gene = None
+ for label, probe in zip(labels, fg_bad_probes):
+ if probe.gene == last_gene:
+ gene = ' "'
+ else:
+ gene = probe.gene
+ last_gene = gene
+ if 'rmask' in probes:
+ logging.info(" %s %s coverage=%.3f spread=%.3f rmask=%.3f",
+ gene.ljust(gene_cols), label.ljust(chrom_cols),
+ probe.log2, probe.spread, probe.rmask)
+ else:
+ logging.info(" %s %s coverage=%.3f spread=%.3f",
+ gene.ljust(gene_cols), label.ljust(chrom_cols),
+ probe.log2, probe.spread)
+
+ # Count the number of BG probes dropped, too (names are all "Background")
+ bg_bad_probes = bad_probes[~fg_index]
+ if len(bg_bad_probes) > 0:
+ bad_pct = 100 * len(bg_bad_probes) / sum(probes['gene'] == 'Background')
+ logging.info("Antitargets: %d (%s) bins failed filters",
+ len(bg_bad_probes), "%.4f" % bad_pct + '%')
+
+
+def get_fasta_stats(probes, fa_fname):
+ """Calculate GC and RepeatMasker content of each bin in the FASTA genome."""
+ fa_coords = zip(probes.chromosome, probes.start, probes.end)
+ logging.info("Calculating GC and RepeatMasker content in %s ...", fa_fname)
+ gc_rm_vals = [calculate_gc_lo(subseq)
+ for subseq in ngfrills.fasta_extract_regions(fa_fname,
+ fa_coords)]
+ gc_vals, rm_vals = zip(*gc_rm_vals)
+ return np.asfarray(gc_vals), np.asfarray(rm_vals)
+
+
+def calculate_gc_lo(subseq):
+ """Calculate the GC and lowercase (RepeatMasked) content of a string."""
+ cnt_at_lo = subseq.count('a') + subseq.count('t')
+ cnt_at_up = subseq.count('A') + subseq.count('T')
+ cnt_gc_lo = subseq.count('g') + subseq.count('c')
+ cnt_gc_up = subseq.count('G') + subseq.count('C')
+ tot = float(cnt_gc_up + cnt_gc_lo + cnt_at_up + cnt_at_lo)
+ if not tot:
+ return 0.0, 0.0
+ frac_gc = (cnt_gc_lo + cnt_gc_up) / tot
+ frac_lo = (cnt_at_lo + cnt_gc_lo) / tot
+ return frac_gc, frac_lo
+
+
+def reference2regions(reference, coord_only=False):
+ """Extract iterables of target and antitarget regions from a reference.
+
+ Like loading two BED files with ngfrills.parse_regions.
+ """
+ cna2rows = (_cna2coords if coord_only else _cna2regions)
+ return map(cna2rows, _ref_split_targets(reference))
+
+
+def _cna2coords(cnarr):
+ """Extract the coordinate columns from a CopyNumberArray"""
+ return zip(cnarr['chromosome'], cnarr['start'], cnarr['end'])
+
+
+def _cna2regions(cnarr):
+ """Extract the region columns (including genes) from a CopyNumberArray"""
+ return zip(cnarr['chromosome'], cnarr['start'], cnarr['end'], cnarr['gene'])
+
+
+def _ref_split_targets(ref_arr):
+ """Split reference into 2 sub-arrays of targets/antitargets."""
+ is_bg = (ref_arr['gene'] == 'Background')
+ targets = ref_arr[~is_bg]
+ antitargets = ref_arr[is_bg]
+ return targets, antitargets
diff --git a/cnvlib/reports.py b/cnvlib/reports.py
new file mode 100644
index 0000000..c8cce1a
--- /dev/null
+++ b/cnvlib/reports.py
@@ -0,0 +1,111 @@
+"""Supporting functions for the text/tabular-reporting commands.
+
+Namely: breaks, gainloss.
+"""
+from __future__ import absolute_import, division
+import collections
+import math
+import sys
+
+from . import metrics, params
+
+iteritems = (dict.iteritems if sys.version_info[0] < 3 else dict.items)
+
+
+# _____________________________________________________________________________
+# breaks
+
+def get_gene_intervals(all_probes, ignore=params.IGNORE_GENE_NAMES):
+ """Tally genomic locations of each targeted gene.
+
+ Return a dict of chromosomes to a list of tuples: (gene name, start, end).
+ """
+ ignore += ("Background",)
+ # Tally the start & end points for each targeted gene; group by chromosome
+ gene_probes = collections.defaultdict(lambda: collections.defaultdict(list))
+ for row in all_probes:
+ gname = str(row.gene)
+ if gname not in ignore:
+ gene_probes[row.chromosome][gname].append(row)
+ # Condense into a single interval for each gene
+ intervals = collections.defaultdict(list)
+ for chrom, gp in iteritems(gene_probes):
+ for gene, probes in iteritems(gp):
+ starts = sorted(row.start for row in probes)
+ end = max(row.end for row in probes)
+ intervals[chrom].append((gene, starts, end))
+ intervals[chrom].sort(key=lambda gse: gse[1])
+ return intervals
+
+
+def get_breakpoints(intervals, segments, min_probes):
+ """Identify CBS segment breaks within the targeted intervals."""
+ breakpoints = []
+ for i, curr_row in enumerate(segments[:-1]):
+ curr_chrom = curr_row.chromosome
+ curr_end = curr_row.end
+ next_row = segments[i + 1]
+ # Skip if this segment is the last (or only) one on this chromosome
+ if next_row.chromosome != curr_chrom:
+ continue
+ for gname, gstarts, gend in intervals[curr_chrom]:
+ if gstarts[0] < curr_end < gend:
+ probes_left = sum(s < curr_end for s in gstarts)
+ probes_right = sum(s >= curr_end for s in gstarts)
+ if probes_left >= min_probes and probes_right >= min_probes:
+ breakpoints.append(
+ (gname, curr_chrom, int(math.ceil(curr_end)),
+ next_row.log2 - curr_row.log2,
+ probes_left, probes_right))
+ breakpoints.sort(key=lambda row: (min(row[4], row[5]), abs(row[3])),
+ reverse=True)
+ return breakpoints
+
+
+# _____________________________________________________________________________
+# gainloss
+
+def gainloss_by_gene(probes, threshold, skip_low=False):
+ """Identify genes where average bin copy ratio value exceeds `threshold`.
+
+ NB: Must shift sex-chromosome values beforehand with shift_xx,
+ otherwise all chrX/chrY genes may be reported gained/lost.
+ """
+ for gene, chrom, start, end, coverage, nprobes in group_by_genes(probes,
+ skip_low):
+ if abs(coverage) >= threshold:
+ yield (gene, chrom, start, end, coverage, nprobes)
+
+
+def gainloss_by_segment(probes, segments, threshold, skip_low=False):
+ """Identify genes where segmented copy ratio exceeds `threshold`.
+
+ NB: Must shift sex-chromosome values beforehand with shift_xx,
+ otherwise all chrX/chrY genes may be reported gained/lost.
+ """
+ for segment, subprobes in probes.by_ranges(segments):
+ if abs(segment.log2) >= threshold:
+ for (gene, chrom, start, end, _coverage, nprobes
+ ) in group_by_genes(subprobes, skip_low):
+ yield (gene, chrom, start, end, segment.log2, nprobes)
+
+
+# TODO consolidate with CNA.squash_genes
+def group_by_genes(probes, skip_low):
+ """Group probe and coverage data by gene.
+
+ Return an iterable of genes, in chromosomal order, associated with their
+ location and coverages:
+
+ [(gene, chrom, start, end, [coverages]), ...]
+ """
+ for gene, rows in probes.by_gene():
+ if gene == 'Background':
+ continue
+ chrom = rows[0, 'chromosome']
+ start = rows[0, 'start']
+ end = rows[len(rows)-1, 'end']
+ segmean = metrics.segment_mean(rows)
+ if segmean is not None:
+ nprobes = len(rows)
+ yield gene, chrom, start, end, segmean, nprobes
diff --git a/cnvlib/segmentation/__init__.py b/cnvlib/segmentation/__init__.py
new file mode 100644
index 0000000..0cbda35
--- /dev/null
+++ b/cnvlib/segmentation/__init__.py
@@ -0,0 +1,211 @@
+"""Segmentation of copy number values."""
+from __future__ import absolute_import, division
+import logging
+import math
+import os.path
+import tempfile
+
+import numpy as np
+import pandas as pd
+
+from .. import core, ngfrills, params, smoothing, vary
+from ..cnary import CopyNumArray as CNA
+from . import cbs, flasso, haar
+
+from Bio._py3k import StringIO
+
+
+def do_segmentation(cnarr, method, threshold=None, variants=None,
+ skip_low=False, skip_outliers=10,
+ save_dataframe=False, rlibpath=None):
+ """Infer copy number segments from the given coverage table."""
+ filtered_cn = cnarr
+ if skip_low:
+ before = len(filtered_cn)
+ filtered_cn = filtered_cn.drop_low_coverage()
+ logging.info("Dropped %d low-coverage bins", before - len(filtered_cn))
+ if skip_outliers:
+ filtered_cn = drop_outliers(filtered_cn, 50, skip_outliers)
+
+ if method == 'haar':
+ threshold = threshold or 0.001
+ segarr = haar.segment_haar(filtered_cn, threshold)
+ segarr['gene'], segarr['weight'] = transfer_names_weights(segarr, cnarr)
+
+ elif method in ('cbs', 'flasso'):
+ # Run R scripts to calculate copy number segments
+ if method == 'cbs':
+ rscript = cbs.CBS_RSCRIPT
+ threshold = threshold or 0.0001
+ elif method == 'flasso':
+ rscript = flasso.FLASSO_RSCRIPT
+ threshold = threshold or 0.005
+
+ with tempfile.NamedTemporaryFile(suffix='.cnr') as tmp:
+ filtered_cn.data.to_csv(tmp, index=False, sep='\t',
+ float_format='%.6g')
+ tmp.flush()
+ script_strings = {
+ 'probes_fname': tmp.name,
+ 'sample_id': cnarr.sample_id,
+ 'threshold': threshold,
+ 'rlibpath': ('.libPaths(c("%s"))' % rlibpath if rlibpath else ''),
+ }
+ with ngfrills.temp_write_text(rscript % script_strings) as script_fname:
+ seg_out = ngfrills.call_quiet('Rscript', script_fname)
+ # ENH: run each chromosome separately
+ # ENH: run each chrom. arm separately (via knownsegs)
+ segarr = cnarr.as_dataframe(seg2cns(seg_out))
+ segarr.sort_columns()
+
+ if method == 'flasso':
+ segarr = squash_segments(segarr)
+
+ segarr = repair_segments(segarr, cnarr)
+
+ else:
+ raise ValueError("Unknown method %r" % method)
+
+ if variants:
+ # Re-segment the variant allele freqs within each segment
+ newsegs = [haar.variants_in_segment(subvarr, segment, 0.01 * threshold)
+ for segment, subvarr in variants.by_ranges(segarr)]
+ segarr = segarr.as_dataframe(pd.concat(newsegs))
+ segarr.sort_columns()
+ # TODO fix ploidy on allosomes
+ allelics = vary._allele_specific_copy_numbers(segarr, variants)
+ segarr.data = pd.concat([segarr.data, allelics], axis=1, copy=False)
+
+ segarr['gene'], segarr['weight'] = transfer_names_weights(segarr, cnarr)
+
+ if save_dataframe:
+ return segarr, seg_out
+ else:
+ return segarr
+
+
+def drop_outliers(cnarr, width, factor):
+ """Drop outlier bins with log2 ratios too far from the trend line.
+
+ Outliers are the log2 values `factor` times the 90th quantile of absolute
+ deviations from the rolling average, within a window of given `width`. The
+ 90th quantile is about 1.97 standard deviations if the log2 values are
+ Gaussian, so this is similar to calling outliers `factor` * 1.97 standard
+ deviations from the rolling mean. For a window size of 50, the breakdown
+ point is 2.5 outliers within a window, which is plenty robust for our needs.
+ """
+ outlier_mask = np.concatenate([
+ smoothing.rolling_outlier_quantile(subarr['log2'], width, .95, factor)
+ for _chrom, subarr in cnarr.by_chromosome()])
+ n_outliers = outlier_mask.sum()
+ if n_outliers:
+ logging.info("Dropped %d outlier bins:\n%s%s",
+ n_outliers,
+ cnarr[outlier_mask].data.head(20),
+ "\n..." if n_outliers > 20 else "")
+ else:
+ logging.info("No outlier bins")
+ return cnarr[~outlier_mask]
+
+
+def transfer_names_weights(segments, cnarr, ignore=params.IGNORE_GENE_NAMES):
+ """Copy gene names from `cnarr` to the segmented `segarr`.
+
+ Segment name is the comma-separated list of bin gene names.
+ """
+ ignore += ("Background",)
+ segnames = ['-'] * len(segments)
+ segweights = np.zeros(len(segments))
+ for i, (_seg, subprobes) in enumerate(cnarr.by_ranges(segments)):
+ segweights[i] = subprobes['weight'].sum()
+ subgenes = [g for g in pd.unique(subprobes['gene']) if g not in ignore]
+ if subgenes:
+ segnames[i] = ",".join(subgenes)
+ return segnames, segweights
+
+
+def seg2cns(seg_text):
+ """Convert R dataframe contents (SEG) to our native tabular format.
+
+ Return a pandas.Dataframe with CNA columns.
+ """
+ try:
+ table = pd.read_table(StringIO(seg_text), comment='[')
+ except pd.parser.CParserError:
+ raise ValueError("Unexpected dataframe contents:\n%s" % (seg_text))
+ if len(table.columns) == 6:
+ table.columns = ["sample_id", "chromosome", "start", "end", "probes",
+ "log2"]
+ elif len(table.columns) == 5:
+ table.columns = ["sample_id", "chromosome", "start", "end", "log2"]
+ else:
+ raise ValueError("Segmentation output is not valid SEG format:\n"
+ + seg_text)
+ del table["sample_id"]
+ table["start"] = [int(math.ceil(float(val))) for val in table["start"]]
+ table["end"] = [int(math.ceil(float(val))) for val in table["end"]]
+ table["gene"] = '-'
+ return table
+
+
+def squash_segments(seg_pset):
+ """Combine contiguous segments."""
+ curr_chrom = None
+ curr_start = None
+ curr_end = None
+ curr_genes = []
+ curr_val = None
+ curr_cnt = 0
+ squashed_rows = []
+ for row in seg_pset:
+ if row.chromosome == curr_chrom and row.log2 == curr_val:
+ # Continue the current segment
+ curr_end = row.end
+ curr_genes.append(row.gene)
+ curr_cnt += 1
+ else:
+ # Segment break
+ # Finish the current segment
+ if curr_cnt:
+ squashed_rows.append((curr_chrom, curr_start, curr_end,
+ ",".join(pd.unique(curr_genes)),
+ curr_val, curr_cnt))
+ # Start a new segment
+ curr_chrom = row.chromosome
+ curr_start = row.start
+ curr_end = row.end
+ curr_genes = []
+ curr_val = row.log2
+ curr_cnt = 1
+ # Remainder
+ squashed_rows.append((curr_chrom, curr_start, curr_end,
+ ",".join(pd.unique(curr_genes)),
+ curr_val, curr_cnt))
+ return seg_pset.as_rows(squashed_rows)
+
+
+def repair_segments(segments, orig_probes):
+ """Post-process segmentation output.
+
+ 1. Ensure every chromosome has at least one segment.
+ 2. Ensure first and last segment ends match 1st/last bin ends
+ (but keep log2 as-is).
+ 3. Store probe-level gene names, comma-separated, as the segment name.
+ """
+ segments = segments.copy()
+ extra_segments = []
+ # Adjust segment endpoints on each chromosome
+ for chrom, subprobes in orig_probes.by_chromosome():
+ chr_seg_idx = np.where(segments.chromosome == chrom)[0]
+ orig_start = subprobes[0, 'start']
+ orig_end = subprobes[len(subprobes)-1, 'end']
+ if len(chr_seg_idx):
+ segments[chr_seg_idx[0], 'start'] = orig_start
+ segments[chr_seg_idx[-1], 'end'] = orig_end
+ else:
+ null_segment = (chrom, orig_start, orig_end, "-", 0.0, 0)
+ extra_segments.append(null_segment)
+ if extra_segments:
+ segments.add(segments.as_rows(extra_segments))
+ # ENH: Recalculate segment means here instead of in R
+ return segments
diff --git a/cnvlib/segmentation/cbs.py b/cnvlib/segmentation/cbs.py
new file mode 100644
index 0000000..5e70d97
--- /dev/null
+++ b/cnvlib/segmentation/cbs.py
@@ -0,0 +1,73 @@
+CBS_RSCRIPT = """\
+#!/usr/bin/env Rscript
+
+# Calculate copy number segmentation by CBS.
+# Input: log2 coverage data in Nexus 'basic' format
+# Output: the CBS data table
+
+%(rlibpath)s
+library('PSCBS') # Requires: R.utils, R.oo, R.methodsS3
+
+write("Loading probe coverages into a data frame", stderr())
+tbl = read.delim("%(probes_fname)s")
+chrom_rle = rle(as.character(tbl$chromosome))
+chrom_names = chrom_rle$value
+chrom_lengths = chrom_rle$lengths
+chrom_ids = rep(1:length(chrom_names), chrom_lengths)
+if (is.null(tbl$weight)) {
+ cna = data.frame(chromosome=chrom_ids, x=tbl$start, y=tbl$log2)
+} else {
+ cna = data.frame(chromosome=chrom_ids, x=tbl$start, y=tbl$log2, w=tbl$weight)
+}
+
+write("Pre-processing the probe data for segmentation", stderr())
+# Find and exclude the centromere of each chromosome
+largegaps = findLargeGaps(cna, minLength=1e6)
+if (is.null(largegaps)) {
+ knownsegs = NULL
+} else {
+ # Choose the largest gap in each chromosome and only omit that
+ rows_to_keep = c()
+ for (i in 1:length(chrom_names)) {
+ curr_chrom_mask = (largegaps$chromosome == i)
+ if (sum(curr_chrom_mask)) {
+ best = which(
+ curr_chrom_mask &
+ (largegaps$length == max(largegaps[curr_chrom_mask,]$length))
+ )
+ rows_to_keep = c(rows_to_keep, best)
+ }
+ }
+ knownsegs = gapsToSegments(largegaps[rows_to_keep,])
+}
+
+write("Segmenting the probe data", stderr())
+fit = segmentByCBS(cna, alpha=%(threshold)g, undo=0, min.width=2,
+ joinSegments=FALSE, knownSegments=knownsegs, seed=0xA5EED)
+
+write("Setting segment endpoints to original bin start/end positions", stderr())
+write("and recalculating segment means with bin weights", stderr())
+for (idx in 1:nrow(fit$output)) {
+ if (!is.na(fit$segRows$startRow[idx])) {
+ start_bin = fit$segRows$startRow[idx]
+ end_bin = fit$segRows$endRow[idx]
+ fit$output$start[idx] = tbl$start[start_bin]
+ fit$output$end[idx] = tbl$end[end_bin]
+ fit$output$mean[idx] = weighted.mean(tbl$log2[start_bin:end_bin],
+ tbl$weight[start_bin:end_bin])
+ }
+}
+
+write("Restoring the original chromosome names", stderr())
+fit$output$sampleName = '%(sample_id)s'
+out = na.omit(fit$output) # Copy for lookup in the loop
+out2 = na.omit(fit$output) # Copy to modify
+for (i in 1:length(chrom_names)) {
+ out2[out$chromosome == i,]$chromosome = chrom_names[i]
+}
+
+write("Printing the CBS table to standard output", stderr())
+write.table(out2, '', sep='\t', row.names=FALSE)
+"""
+
+
diff --git a/cnvlib/segmentation/flasso.py b/cnvlib/segmentation/flasso.py
new file mode 100644
index 0000000..610cf2a
--- /dev/null
+++ b/cnvlib/segmentation/flasso.py
@@ -0,0 +1,28 @@
+FLASSO_RSCRIPT = """\
+#!/usr/bin/env Rscript
+
+# Calculate copy number segmentation by CBS.
+# Input: log2 coverage data in Nexus 'basic' format
+# Output: the CBS data table
+
+%(rlibpath)s
+library('cghFLasso')
+
+tbl <- read.delim("%(probes_fname)s")
+# Ignore low-coverage probes
+positions <- (tbl$start + tbl$end) * 0.5
+
+write("Segmenting the probe data", stderr())
+fit <- cghFLasso(tbl$log2, FDR=%(threshold)g)
+
+# Reformat the output table as SEG
+outtable <- data.frame(sample="%(sample_id)s",
+ chromosome=tbl$chromosome,
+ start=tbl$start,
+ end=tbl$end,
+ nprobes=1,
+ value=fit$Esti.CopyN)
+
+write("Printing the segment table to standard output", stderr())
+write.table(outtable, '', sep='\t', row.names=FALSE)
+"""
diff --git a/cnvlib/segmentation/haar.py b/cnvlib/segmentation/haar.py
new file mode 100644
index 0000000..c7bd5ac
--- /dev/null
+++ b/cnvlib/segmentation/haar.py
@@ -0,0 +1,571 @@
+#!/usr/bin/env python
+
+"""Probe segmentation by convolving with the Haar wavelet.
+
+The basic HaarSeg algorithm:
+
+* Apply the undecimated discrete wavelet transform (UDWT) on the data, using the
+ Haar wavelet.
+* Select a set of detail subbands from the transform {LMIN, LMIN+1, ..., LMAX}.
+* Find the local maxima of the selected detail subbands.
+* Threshold the maxima of each subband separately, using an FDR thresholding
+ procedure.
+* Unify selected maxima from all the subbands to create a list of significant
+ breakpoints in the data.
+* Reconstruct the segmentation result from the list of significant breakpoints.
+
+HaarSeg segmentation is based on detecting local maxima in the wavelet domain,
+using Haar wavelet. The main algorithm parameter is breaksFdrQ, which controls
+the sensitivity of the segmentation result. This function supports the optional
+use of weights (also known as quality of measurments) and raw measurments. We
+recommend using both extentions where possible, as it greatly improves the
+segmentation result.
+
+"""
+from __future__ import absolute_import, division, print_function
+
+import logging
+import math
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+# from .. import params
+
+
+def segment_haar(cnarr, fdr_q):
+ """Do segmentation for CNVkit.
+
+ Calculate copy number segmentation by HaarSeg
+ (http://haarseg.r-forge.r-project.org/)
+ Input: log2 coverage data in Nexus 'basic' format
+ Output: the CBS data table
+
+ """
+ # Segment each chromosome individually
+ # ENH - skip large gaps (segment chrom. arms separately)
+ chrom_tables = [one_chrom(subprobes, fdr_q, chrom)
+ for chrom, subprobes in cnarr.by_chromosome()]
+ segarr = cnarr.as_dataframe(pd.concat(chrom_tables))
+ segarr.sort_columns()
+ return segarr
+
+
+def one_chrom(cnarr, fdr_q, chrom):
+ logging.debug("Segmenting %s", chrom)
+ segtable = haarSeg(np.asarray(cnarr['log2']),
+ fdr_q,
+ W=(np.asarray(cnarr['weight'])
+ if 'weight' in cnarr
+ else None))
+ table = pd.DataFrame({
+ 'chromosome': chrom,
+ 'start': np.asarray(cnarr['start']).take(segtable['start']),
+ 'end': np.asarray(cnarr['end']).take(segtable['end']),
+ 'log2': segtable['mean'],
+ 'gene': '-',
+ 'probes': segtable['size'],
+ })
+ return table
+
+
+def variants_in_segment(varr, segment, fdr_q):
+ if len(varr):
+ values = varr.mirrored_baf()
+ segtable = haarSeg(values,
+ fdr_q,
+ W=None) # weight by sqrt(DP)?
+ else:
+ values = []
+ segtable = []
+ if segtable and len(segtable['start']) > 1:
+ logging.info("Segmented on allele freqs in %s:%d-%d",
+ segment.chromosome, segment.start, segment.end)
+ # Ensure breakpoint locations make sense
+ # - Keep original segment start, end positions
+ # - Place breakpoints midway between SNVs, I guess?
+ gap_rights = np.asarray(varr['start']).take(segtable['start'][1:])
+ gap_lefts = np.asarray(varr['end']).take(segtable['end'][:-1])
+ mid_breakpoints = [(left + right) // 2
+ for left, right in zip(gap_lefts, gap_rights)]
+ starts = np.concatenate([[segment.start], mid_breakpoints])
+ ends = np.concatenate([mid_breakpoints, [segment.end]])
+ table = pd.DataFrame({
+ 'chromosome': segment.chromosome,
+ 'start': starts,
+ 'end': ends,
+ # 'baf': segtable['mean'],
+ 'gene': '-', #segment.gene,
+ 'log2': segment.log2,
+ 'probes': segtable['size'],
+ # 'weight': segment.weight * segtable['size'] / segment.size,
+ })
+ else:
+ table = pd.DataFrame({
+ 'chromosome': segment.chromosome,
+ 'start': segment.start,
+ 'end': segment.end,
+ # 'baf': np.median(values),
+ 'gene': '-', #segment.gene,
+ 'log2': segment.log2,
+ 'probes': segment.probes,
+ # 'weight': segment.weight,
+ }, index=[0])
+
+ return table
+
+
+
+# ---- from HaarSeg R code -- the API ----
+
+def haarSeg(I, breaksFdrQ,
+ W=None,
+ rawI=None,
+ haarStartLevel=1,
+ haarEndLevel=5):
+ r"""Perform segmentation according to the HaarSeg algorithm.
+
+ Arguments:
+
+ I
+ A 1D array of log-ratio values, sorted according to their genomic
+ location.
+ W
+ Weight matrix, corresponding to quality of measurement, with values
+ :math:`1/(\sigma^2)`. Must have the same size as I.
+ rawI
+ The minimum between the raw test-sample and control-sample coverages
+ (before applying log ratio, but after any background reduction and/or
+ normalization). These raw red / green measurments are used to detect
+ low-value probes, which are more sensitive to noise.
+ Used for the non-stationary variance compensation.
+ Must have the same size as I.
+ breaksFdrQ
+ The FDR q parameter. This value should lie between 0 and 0.5. The
+ smaller this value is, the less sensitive the segmentation result will
+ be.
+ For example, we will detect fewer segmentation breaks when using Q =
+ 1e-4, compared to when using Q = 1e-3.
+ Common used values are 1e-2, 1e-3, 1e-4.
+ haarStartLevel
+ The detail subband from which we start to detect peaks. The higher this
+ value is, the less sensitive we are to short segments. The default is
+ value is 1, corresponding to segments of 2 probes.
+ haarEndLevel
+ The detail subband until which we use to detect peaks. The higher this
+ value is, the more sensitive we are to large trends in the data. This
+ value DOES NOT indicate the largest possible segment that can be
+ detected. The default is value is 5, corresponding to step of 32 probes
+ in each direction.
+
+
+ Returns:
+
+ A tuple of two elements:
+ 1. Segments result table, a list of tuples:
+ (segment start index, segment size, segment value)
+ 2. Segmented signal array (same size as I)
+
+ Source: haarSeg.R
+ """
+ def med_abs_diff(diff_vals):
+ """Median absolute deviation, with deviations given."""
+ if len(diff_vals) == 0:
+ return 0.
+ return np.median(np.abs(diff_vals)) * 1.4826
+
+ diffI = HaarConv(I, None, 1)
+ if rawI:
+ # Non-stationary variance empirical threshold set to 50
+ NSV_TH = 50
+ varMask = (rawI < NSV_TH)
+ pulseSize = 2
+ diffMask = (PulseConv(varMask, pulseSize) >= .5)
+ peakSigmaEst = med_abs_diff(diffI[~diffMask])
+ noisySigmaEst = med_abs_diff(diffI[diffMask])
+ else:
+ peakSigmaEst = med_abs_diff(diffI)
+
+ breakpoints = np.array([], dtype=np.int_)
+ for level in range(haarStartLevel, haarEndLevel+1):
+ stepHalfSize = 2 ** level
+ convRes = HaarConv(I, W, stepHalfSize)
+ peakLoc = FindLocalPeaks(convRes)
+ logging.debug("Found %d peaks at level %d", len(peakLoc), level)
+
+ if rawI:
+ pulseSize = 2 * stepHalfSize
+ convMask = (PulseConv(varMask, pulseSize) >= .5)
+ sigmaEst = (1 - convMask) * peakSigmaEst + convMask * noisySigmaEst
+ convRes /= sigmaEst
+ peakSigmaEst = 1.
+
+ T = FDRThres(convRes[peakLoc], breaksFdrQ, peakSigmaEst)
+ # Keep only the peak values where the signal amplitude is large enough.
+ addonPeaks = np.extract(np.abs(convRes.take(peakLoc)) >= T, peakLoc)
+ breakpoints = UnifyLevels(breakpoints, addonPeaks, 2 ** (level - 1))
+
+ logging.debug("Found %d breakpoints: %s", len(breakpoints), breakpoints)
+
+ # Translate breakpoints to segments
+ segs = SegmentByPeaks(I, breakpoints, W)
+ segSt = np.insert(breakpoints, 0, 0)
+ segEd = np.append(breakpoints, len(I))
+ return {'start': segSt,
+ 'end': segEd - 1,
+ 'size': segEd - segSt,
+ 'mean': segs[segSt]}
+
+
+def FDRThres(x, q, stdev):
+ """False discovery rate (FDR) threshold."""
+ M = len(x)
+ if M < 2:
+ return 0
+
+ m = np.arange(1, M+1) / M
+ x_sorted = np.sort(np.abs(x))[::-1]
+ p = 2 * (1 - stats.norm.cdf(x_sorted, stdev)) # like R "pnorm"
+ # Get the largest index for which p <= m*q
+ indices = np.nonzero(p <= m * q)[0]
+ if len(indices):
+ T = x_sorted[indices[-1]]
+ else:
+ logging.debug("No passing p-values: min p=%.4g, min m=%.4g, q=%s",
+ p[0], m[0], q)
+ T = x_sorted[0] + 1e-16 # ~= 2^-52, like MATLAB "eps"
+ return T
+
+
+def SegmentByPeaks(data, peaks, weights=None):
+ """Average the values of the probes within each segment.
+
+ `data` : the probe array values
+ `peaks` : array of copy number breakpoints
+
+ Source: SegmentByPeaks.R
+ """
+ segs = np.zeros_like(data)
+ for seg_start, seg_end in zip(np.insert(peaks, 0, 0),
+ np.append(peaks, len(data))):
+ if weights is not None and weights[seg_start:seg_end].sum() > 0:
+ # Weighted mean of individual probe values
+ val = np.average(data[seg_start:seg_end],
+ weights=weights[seg_start:seg_end])
+ else:
+ # Unweighted mean of individual probe values
+ val = np.mean(data[seg_start:seg_end])
+ segs[seg_start:seg_end] = val
+ return segs
+
+
+
+# ---- from HaarSeg C code -- the core ----
+
+# --- HaarSeg.h
+def HaarConv(signal, #const double * signal,
+ weight, #const double * weight,
+ stepHalfSize, #int stepHalfSize,
+ ):
+ """Convolve haar wavelet function with a signal, applying circular padding.
+
+ Params:
+
+ signal: const array of floats
+ weight: const array of floats (optional)
+ stepHalfSize: int
+
+ Output:
+ result: array of floats
+
+ Source: HaarSeg.c
+ """
+ signalSize = len(signal)
+ if stepHalfSize > signalSize:
+ # XXX TODO handle this endcase
+ # raise ValueError("stepHalfSize (%s) > signalSize (%s)"
+ # % (stepHalfSize, signalSize))
+ logging.debug("Error?: stepHalfSize (%s) > signalSize (%s)",
+ stepHalfSize, signalSize)
+ return np.zeros(signalSize, dtype=np.float_)
+
+ result = np.zeros(signalSize, dtype=np.float_)
+ if weight is not None:
+ # Init weight sums
+ highWeightSum = weight[:stepHalfSize].sum()
+ # highSquareSum = np.exp2(weight[:stepHalfSize]).sum()
+ highNonNormed = (weight[:stepHalfSize] * signal[:stepHalfSize]).sum()
+ # Circular padding
+ lowWeightSum = highWeightSum
+ # lowSquareSum = highSquareSum
+ lowNonNormed = -highNonNormed
+
+ # ENH: vectorize this loop (it's the performance hotspot)
+ for k in range(1, signalSize):
+ highEnd = k + stepHalfSize - 1
+ if highEnd >= signalSize:
+ highEnd = signalSize - 1 - (highEnd - signalSize)
+ lowEnd = k - stepHalfSize - 1
+ if lowEnd < 0:
+ lowEnd = -lowEnd - 1
+
+ if weight is None:
+ result[k] = result[k-1] + signal[highEnd] + signal[lowEnd] - 2*signal[k-1]
+ else:
+ lowNonNormed += signal[lowEnd] * weight[lowEnd] - signal[k-1] * weight[k-1]
+ highNonNormed += signal[highEnd] * weight[highEnd] - signal[k-1] * weight[k-1]
+ lowWeightSum += weight[k-1] - weight[lowEnd]
+ highWeightSum += weight[highEnd] - weight[k-1]
+ # lowSquareSum += weight[k-1] * weight[k-1] - weight[lowEnd] * weight[lowEnd]
+ # highSquareSum += weight[highEnd] * weight[highEnd] - weight[k-1] * weight[k-1]
+ result[k] = math.sqrt(stepHalfSize / 2) * (lowNonNormed / lowWeightSum +
+ highNonNormed / highWeightSum)
+
+ if weight is None:
+ stepNorm = math.sqrt(2. * stepHalfSize)
+ result[1:signalSize] /= stepNorm
+
+ return result
+
+
+def FindLocalPeaks(signal, #const double * signal,
+ # peakLoc, #int * peakLoc
+ ):
+ """Find local maxima on positive values, local minima on negative values.
+
+ First and last index are never considered extramum.
+
+ Parameters:
+
+ signal: const array of floats
+
+ Output:
+ peakLoc: array of ints
+
+ Source: HaarSeg.c
+ """
+ # use numpy.diff to simplify? argmax, argmin?
+ maxSuspect = minSuspect = None
+ peakLoc = []
+ for k in range(1, len(signal) - 1):
+ sig_prev, sig_curr, sig_next = signal[k-1:k+2]
+ if sig_curr > 0:
+ # Look for local maxima
+ if (sig_curr > sig_prev) and (sig_curr > sig_next):
+ peakLoc.append(k)
+ elif (sig_curr > sig_prev) and (sig_curr == sig_next):
+ maxSuspect = k
+ elif (sig_curr == sig_prev) and (sig_curr > sig_next):
+ # Take the first in a series of equal values
+ if maxSuspect is not None:
+ peakLoc.append(maxSuspect)
+ maxSuspect = None
+ elif (sig_curr == sig_prev) and (sig_curr < sig_next):
+ maxSuspect = None
+
+ elif sig_curr < 0:
+ # Look for local maxima
+ if (sig_curr < sig_prev) and (sig_curr < sig_next):
+ peakLoc.append(k)
+ elif (sig_curr < sig_prev) and (sig_curr == sig_next):
+ minSuspect = k
+ elif (sig_curr == sig_prev) and (sig_curr < sig_next):
+ if minSuspect is not None:
+ peakLoc.append(minSuspect)
+ minSuspect = None
+ elif (sig_curr == sig_prev) and (sig_curr > sig_next):
+ minSuspect = None
+
+ return np.array(peakLoc, dtype=np.int_)
+
+
+def UnifyLevels(baseLevel, #const int * baseLevel,
+ addonLevel, #const int * addonLevel,
+ windowSize, #int windowSize,
+ # joinedLevel, #int * joinedLevel);
+ ):
+ """Unify several decomposition levels.
+
+ Merge the two lists of breakpoints, but drop addonLevel values that are too
+ close to baseLevel values.
+
+ Parameters:
+
+ baseLevel: const array of ints
+ addonLevel: const array of ints
+ windowSize: int
+
+ Output:
+
+ joinedLevel: array of ints
+
+ Source: HaarSeg.c
+ """
+ if not len(addonLevel):
+ return baseLevel
+
+ # Merge all addon items outside a window around each base item
+ # ENH: do something clever with searchsorted & masks?
+ joinedLevel = []
+ addon_idx = 0
+ for base_elem in baseLevel:
+ while addon_idx < len(addonLevel):
+ addon_elem = addonLevel[addon_idx]
+ if addon_elem < base_elem - windowSize:
+ # Addon is well before this base item -- use it
+ joinedLevel.append(addon_elem)
+ addon_idx += 1
+ elif base_elem - windowSize <= addon_elem <= base_elem + windowSize:
+ # Addon is too close to this base item -- skip it
+ addon_idx += 1
+ else:
+ assert base_elem + windowSize < addon_elem
+ # Addon is well beyond this base item -- keep for the next round
+ break
+ joinedLevel.append(base_elem)
+
+ # Append the remaining addon items beyond the last base item's window
+ last_pos = (baseLevel[-1] + windowSize if len(baseLevel) else -1)
+ while addon_idx < len(addonLevel) and addonLevel[addon_idx] <= last_pos:
+ addon_idx += 1
+ if addon_idx < len(addonLevel):
+ joinedLevel.extend(addonLevel[addon_idx:])
+
+ return np.array(sorted(joinedLevel), dtype=np.int_)
+
+
+def PulseConv(signal, #const double * signal,
+ pulseSize, #int pulseSize,
+ ):
+ """Convolve a pulse function with a signal, applying circular padding to the
+ signal.
+
+ Used for non-stationary variance compensation.
+
+ Parameters:
+
+ signal: const array of floats
+ pulseSize: int
+
+ Output:
+ result: modifiable array of floats
+
+ Source: HaarSeg.c
+ """
+ signalSize = len(signal)
+ if pulseSize > signalSize:
+ # ENH: handle this endcase
+ raise ValueError("pulseSize (%s) > signalSize (%s)"
+ % (pulseSize, signalSize))
+ pulseHeight = 1. / pulseSize
+
+ # Circular padding init
+ result = np.zeros(signalSize, dtype=np.float_)
+ for k in range((pulseSize + 1) // 2):
+ result[0] += signal[k]
+ for k in range(pulseSize // 2):
+ result[0] += signal[k]
+ result[0] *= pulseHeight
+
+ n = 1
+ for k in range(pulseSize // 2,
+ signalSize + (pulseSize // 2) - 1):
+ tail = k - pulseSize
+ if tail < 0:
+ tail = -tail - 1
+ head = k
+ if head >= signalSize:
+ head = signalSize - 1 - (head - signalSize)
+ result[n] = result[n-1] + ((signal[head] - signal[tail]) * pulseHeight)
+ n += 1
+
+ return result
+
+
+# XXX Apply afterward to the segmentation result? (not currently used)
+def AdjustBreaks(signal, #const double * signal,
+ peakLoc, #const int * peakLoc,
+ ):
+ """Improve localization of breaks. Suboptimal, but linear-complexity.
+
+ We try to move each break 1 sample left/right, choosing the offset which
+ leads to minimum data error.
+
+ Parameters:
+
+ signal: const array of floats
+ peakLoc: const array of ints
+
+ Source: HaarSeg.c
+ """
+ newPeakLoc = peakLoc.copy()
+ for k, npl_k in enumerate(newPeakLoc):
+ # Calculating width of segments around the breakpoint
+ n1 = (npl_k if k == 0
+ else npl_k - newPeakLoc[k-1])
+ n2 = (len(signal) if k+1 == len(newPeakLoc)
+ else newPeakLoc[k+1])- npl_k
+
+ # Find the best offset for current breakpoint, trying only 1 sample
+ # offset
+ bestScore = float("Inf") # Smaller is better
+ bestOffset = 0
+ for p in (-1, 0, 1):
+ # Pointless to try to remove single-sample segments
+ if (n1 == 1 and p == -1) or (n2 == 1 and p == 1):
+ continue
+
+ signal_n1_to_p = signal[npl_k - n1:npl_k + p]
+ s1 = signal_n1_to_p.sum() / (n1 + p)
+ ss1 = ((signal_n1_to_p - s1)**2).sum()
+
+ signal_p_to_n2 = signal[npl_k + p:npl_k + n2]
+ s2 = signal_p_to_n2.sum() / (n2 - p)
+ ss2 = ((signal_p_to_n2 - s2)**2).sum()
+
+ score = ss1 + ss2
+ if score < bestScore:
+ bestScore = score
+ bestOffset = p
+
+ if bestOffset != 0:
+ newPeakLoc[k] += bestOffset
+
+ return newPeakLoc
+
+
+# Testing
+
+def table2coords(seg_table):
+ """Return x, y arrays for plotting."""
+ x = []
+ y = []
+ for start, size, val in seg_table:
+ x.append(start)
+ x.append(start + size)
+ y.append(val)
+ y.append(val)
+ return x, y
+
+
+if __name__ == '__main__':
+ real_data = np.concatenate((np.zeros(800), np.ones(200),
+ np.zeros(800), .8*np.ones(200), np.zeros(800)))
+ # np.random.seed(0x5EED)
+ noisy_data = real_data + np.random.standard_normal(len(real_data)) * .2
+
+ # # Run using default parameters
+ seg_table = haarSeg(noisy_data, .005)
+
+ logging.info("%s", seg_table)
+
+ from matplotlib import pyplot
+ indices = np.arange(len(noisy_data))
+ pyplot.scatter(indices, noisy_data, alpha=0.2, color='gray')
+ x, y = table2coords(seg_table)
+ pyplot.plot(x, y, color='r', marker='x', lw=2)
+ pyplot.show()
+
+ # # The complete segmented signal
+ # lines(seg.data$Segmented, col="red", lwd=3)
diff --git a/cnvlib/smoothing.py b/cnvlib/smoothing.py
new file mode 100644
index 0000000..2f28621
--- /dev/null
+++ b/cnvlib/smoothing.py
@@ -0,0 +1,226 @@
+"""Signal smoothing functions."""
+from __future__ import absolute_import, division
+
+import math
+import warnings
+
+import numpy as np
+import pandas as pd
+
+from . import metrics
+
+
+def check_inputs(x, width):
+ """Transform width into a half-window size.
+
+ `width` is either a fraction of the length of `x` or an integer size of the
+ whole window. The output half-window size is truncated to the length of `x`
+ if needed.
+ """
+ x = np.asfarray(x)
+ if 0 < width < 1:
+ wing = int(math.ceil(len(x) * width * 0.5))
+ elif width >= 2 and int(width) == width:
+ wing = width // 2
+ else:
+ raise ValueError("width must be between 0 and 1 (got %s)" % width)
+ if wing > len(x):
+ wing = len(x) - 1
+ assert wing > 0, "Wing must be greater than 0 (got %s)" % wing
+ return x, wing
+
+
+def rolling_median(x, width):
+ """Rolling median with mirrored edges."""
+ x, wing = check_inputs(x, width)
+ # Pad the edges of the original array with mirror copies
+ signal = np.concatenate((x[wing-1::-1], x, x[:-wing-1:-1]))
+ with warnings.catch_warnings():
+ # NB: in pandas 0.18+ this function is deprecated
+ warnings.simplefilter("ignore", FutureWarning)
+ rolled = pd.rolling_median(signal, 2 * wing + 1, center=True)
+ return rolled[wing:-wing]
+
+
+def rolling_quantile(x, width, quantile):
+ """Rolling quantile (0--1) with mirrored edges."""
+ x, wing = check_inputs(x, width)
+ # Pad the edges of the original array with mirror copies
+ signal = np.concatenate((x[wing-1::-1], x, x[:-wing-1:-1]))
+ with warnings.catch_warnings():
+ # NB: in pandas 0.18+ this function is deprecated
+ warnings.simplefilter("ignore", FutureWarning)
+ rolled = pd.rolling_quantile(signal, 2 * wing + 1, quantile,
+ center=True)
+ return rolled[wing:-wing]
+
+
+def rolling_std(x, width):
+ """Rolling quantile (0--1) with mirrored edges."""
+ x, wing = check_inputs(x, width)
+ # Pad the edges of the original array with mirror copies
+ signal = np.concatenate((x[wing-1::-1], x, x[:-wing-1:-1]))
+ with warnings.catch_warnings():
+ # NB: in pandas 0.18+ this function is deprecated
+ warnings.simplefilter("ignore", FutureWarning)
+ rolled = pd.rolling_std(signal, 2 * wing + 1, center=True)
+ return rolled[wing:-wing]
+
+
+def smoothed(x, width, do_fit_edges=False):
+ """Smooth the values in `x` with the Kaiser windowed filter.
+
+ See: https://en.wikipedia.org/wiki/Kaiser_window
+
+ Parameters:
+
+ x : array-like
+ 1-dimensional numeric data set.
+ width : float
+ Fraction of x's total length to include in the rolling window (i.e. the
+ proportional window width), or the integer size of the window.
+ """
+ x, wing = check_inputs(x, width)
+ # Pad the edges with mirror-image copies of the array
+ signal = np.concatenate((x[wing-1::-1], x, x[:-wing-1:-1]))
+ # Apply signal smoothing
+ window = np.kaiser(2 * wing + 1, 14)
+ y = np.convolve(window / window.sum(), signal, mode='same')
+ # Chop off the ends of the result so it has the original size
+ y = y[wing:-wing]
+ if do_fit_edges:
+ fit_edges(x, y, wing) # In-place
+ return y
+
+
+def fit_edges(x, y, wing, polyorder=3):
+ """Apply polynomial interpolation to the edges of y, in-place.
+
+ Calculates a polynomial fit (of order `polyorder`) of `x` within a window of
+ width twice `wing`, then updates the smoothed values `y` in the half of the
+ window closest to the edge.
+ """
+ window_length = 2 * wing + 1
+ n = len(x)
+ # Fit each of the two array edges (start and end)
+ _fit_edge(x, y, 0, window_length, 0, wing, polyorder)
+ _fit_edge(x, y, n - window_length, n, n - wing, n, polyorder)
+ # TODO - fix the discontinuities at wing, n-wing
+
+
+def _fit_edge(x, y, window_start, window_stop, interp_start, interp_stop,
+ polyorder):
+ """
+ Given a 1-D array `x` and the specification of a slice of `x` from
+ `window_start` to `window_stop`, create an interpolating polynomial of the
+ sliced sub-array, and evaluate that polynomial from `interp_start` to
+ `interp_stop`. Put the result into the corresponding slice of `y`.
+ """
+ # Get the edge into a (window_length, -1) array.
+ x_edge = x[window_start:window_stop]
+ # Fit the edges. poly_coeffs has shape (polyorder + 1, -1),
+ # where '-1' is the same as in x_edge.
+ poly_coeffs = np.polyfit(np.arange(0, window_stop - window_start),
+ x_edge, polyorder)
+ # Compute the interpolated values for the edge.
+ i = np.arange(interp_start - window_start, interp_stop - window_start)
+ values = np.polyval(poly_coeffs, i)
+ # Put the values into the appropriate slice of y.
+ y[interp_start:interp_stop] = values
+
+
+# Outlier detection
+
+def outlier_iqr(a, c=3.0):
+ """Detect outliers as a multiple of the IQR from the median.
+
+ By convention, "outliers" are points more than 1.5 * IQR from the median,
+ and "extremes" or extreme outliers are those more than 3.0 * IQR.
+ """
+ a = np.asarray(a)
+ dists = np.abs(a - np.median(a))
+ iqr = metrics.interquartile_range(a)
+ return dists > (c * iqr)
+
+
+def outlier_mad_median(a):
+ """MAD-Median rule for detecting outliers.
+
+ Returns: a boolean array of the same size, where outlier indices are True.
+
+ X_i is an outlier if::
+
+ | X_i - M |
+ _____________ > K ~= 2.24
+
+ MAD / 0.6745
+
+ where $K = sqrt( X^2_{0.975,1} )$,
+ the square root of the 0.975 quantile of a chi-squared distribution with 1
+ degree of freedom.
+
+ This is a very robust rule with the highest possible breakdown point of 0.5.
+
+ See:
+
+ - Davies & Gather (1993) The Identification of Multiple Outliers.
+ - Rand R. Wilcox (2012) Introduction to robust estimation and hypothesis
+ testing. Ch.3: Estimating measures of location and scale.
+ """
+ K = 2.24
+
+ a = np.asarray(a)
+ dists = np.abs(a - np.median(a))
+ mad = metrics.median_absolute_deviation(a)
+ return (dists / mad) > K
+
+
+def rolling_outlier_iqr(x, width, c=3.0):
+ """Detect outliers as a multiple of the IQR from the median.
+
+ By convention, "outliers" are points more than 1.5 * IQR from the median (~2
+ SD if values are normally distributed), and "extremes" or extreme outliers
+ are those more than 3.0 * IQR (~4 SD).
+ """
+ if len(x) <= width:
+ return np.zeros(len(x), dtype=np.bool_)
+ dists = x - smoothed(x, width)
+ q_hi = rolling_quantile(dists, width, .75)
+ q_lo = rolling_quantile(dists, width, .25)
+ iqr = q_hi - q_lo
+ outliers = (np.abs(dists) > iqr * c)
+ return outliers
+
+
+def rolling_outlier_quantile(x, width, q, m):
+ """Return a boolean mask of outliers by multiples of a quantile in a window.
+
+ Outliers are the array elements outside `m` times the `q`'th
+ quantile of deviations from the smoothed trend line, as calculated from
+ the trend line residuals. (For example, take the magnitude of the 95th
+ quantile times 5, and mark any elements greater than that value as
+ outliers.)
+
+ This is the smoothing method used in BIC-seq (doi:10.1073/pnas.1110574108)
+ with the parameters width=200, q=.95, m=5 for WGS.
+ """
+ if len(x) <= width:
+ return np.zeros(len(x), dtype=np.bool_)
+ dists = np.abs(x - smoothed(x, width))
+ quants = rolling_quantile(dists, width, q)
+ outliers = (dists > quants * m)
+ return outliers
+
+
+def rolling_outlier_std(x, width, stdevs):
+ """Return a boolean mask of outliers by stdev within a rolling window.
+
+ Outliers are the array elements outside `stdevs` standard deviations from
+ the smoothed trend line, as calculated from the trend line residuals.
+ """
+ if len(x) <= width:
+ return np.zeros(len(x), dtype=np.bool_)
+ dists = x - smoothed(x, width)
+ x_std = rolling_std(dists, width)
+ outliers = (np.abs(dists) > x_std * stdevs)
+ return outliers
diff --git a/cnvlib/target.py b/cnvlib/target.py
new file mode 100644
index 0000000..b23010e
--- /dev/null
+++ b/cnvlib/target.py
@@ -0,0 +1,323 @@
+"""Transform bait intervals into targets more suitable for CNVkit."""
+from __future__ import division, absolute_import
+
+import logging
+import collections
+from itertools import groupby
+
+
+# _____________________________________________________________________________
+# Assign refFlat gene names to target intervals
+# ftp://hgdownload.soe.ucsc.edu/goldenPath/hg19/database/
+
+def add_refflat_names(region_rows, refflat_fname):
+ """Apply RefSeq gene names to a list of targeted regions."""
+ return assign_names(region_rows, refflat_fname)
+
+
+def assign_names(region_rows, refflat_fname, default_name='-'):
+ """Replace the interval gene names with those at the same loc in refFlat.txt
+ """
+ ref_genes = read_refflat_genes(refflat_fname)
+ for (chrom, strand), chr_rows in groupby(sorted(region_rows,
+ key=lambda row: (
+ row[0], # chromosome
+ row[4], # strand
+ row[1], # start
+ row[2])), # end
+ lambda row: (row[0], row[4])):
+ if strand == '.':
+ strand = "+|-"
+ if (chrom, strand) in ref_genes:
+ genes_in_chrom = iter(ref_genes[(chrom, strand)])
+ elif '|' in strand:
+ strands_with_genes = []
+ for std in strand.split('|'):
+ if (chrom, std) in ref_genes:
+ strands_with_genes.extend(ref_genes[(chrom, std)])
+ if strands_with_genes:
+ genes_in_chrom = iter(sorted(strands_with_genes))
+ else:
+ logging.info("Chromosome %s %s not in annotations",
+ chrom, strand)
+ continue
+ else:
+ logging.info("Chromosome %s %s not in annotations",
+ chrom, strand)
+ continue
+
+ ex_start, ex_end, ex_name = next(genes_in_chrom)
+ for row in chr_rows:
+ start, end = row[1:3]
+ if ex_end < start:
+ # Burn through genes until hitting or passing the interval
+ while ex_end < start:
+ try:
+ ex_start, ex_end, ex_name = next(genes_in_chrom)
+ except StopIteration:
+ # Interval is past the last annotated gene in chromosome
+ # Fake it...
+ ex_start, ex_end = end + 1, end + 2
+ ex_name = default_name
+
+ if ex_start > end:
+ # Interval is an unannotated intergenic region (we skipped it)
+ yield (chrom, start, end, default_name)
+ else:
+ assert ex_end > start or ex_start < end
+ # Overlap: Use this gene's name and emit the interval
+ yield (chrom, start, end, ex_name)
+
+
+def read_refflat_genes(fname):
+ """Parse genes; merge those with same name and overlapping regions.
+
+ Returns a dict of: {(chrom, strand): [(gene start, gene end, gene name), ...]}
+ """
+ genedict = collections.defaultdict(list)
+ with open(fname) as genefile:
+ for line in genefile:
+ name, _rx, chrom, strand, start, end, _ex = parse_refflat_line(line)
+ # Skip antisense RNA annotations
+ if name.endswith('-AS1'):
+ continue
+ assert strand in '+-'
+ genedict[name].append((chrom, strand, start, end))
+
+ chromdict = collections.defaultdict(list)
+ for name, locs in genedict.iteritems():
+ locs = list(set(locs))
+ if len(locs) == 1:
+ chrom, strand, start, end = locs[0]
+ chromdict[(chrom, strand)].append((start, end, name))
+ else:
+ # Check locs for overlap; merge if so
+ locs.sort()
+ curr_chrom, curr_strand, curr_start, curr_end = locs[0]
+ curr_start, curr_end = int(curr_start), int(curr_end)
+ for chrom, strand, start, end in locs[1:]:
+ start, end = int(start), int(end)
+ if (chrom != curr_chrom or
+ strand != curr_strand or
+ start > curr_end + 1 or
+ end < curr_start - 1):
+ chromdict[(curr_chrom, curr_strand)].append(
+ (curr_start, curr_end, name))
+ curr_chrom, curr_start, curr_end = chrom, start, end
+ else:
+ curr_start = min(start, curr_start)
+ curr_end = max(end, curr_end)
+ # Emit the final interval in this group
+ chromdict[(curr_chrom, curr_strand)].append(
+ (curr_start, curr_end, name))
+
+ # Finally, sort the keys, and return the results
+ for key in list(chromdict):
+ chromdict[key].sort()
+ return chromdict
+
+
+def parse_refflat_line(line):
+ """Parse one line of refFlat.txt; return relevant info.
+
+ Pair up the exon start and end positions. Add strand direction to each
+ chromosome as a "+"/"-" suffix (it will be stripped off later).
+ """
+ fields = line.split('\t')
+ name, refseq, chrom, strand = fields[:4]
+ start, end = fields[4:6]
+ # start, end = fields[6:8]
+ exon_count, exon_starts, exon_ends = fields[8:11]
+ exons = zip(exon_starts.rstrip(',').split(','),
+ exon_ends.rstrip(',').split(','))
+ assert len(exons) == int(exon_count), (
+ "Exon count mismatch at %s: file says %s, but counted %d intervals"
+ % (name, exon_count, len(exons)))
+ return name, refseq, chrom, strand, int(start), int(end), exons
+
+
+# def cover_introns(regions):
+# """Apply the name of the surrounding gene to intronic targets.
+
+# If an unnamed target is surrounded by targets that have the same name,
+# consider it an intron and apply that gene name to the unnamed target.
+# Otherwise, leave it unnamed ("-").
+# """
+# last_gene = None
+# last_chrom = None
+# queue = []
+# for chrom, start, end, name in regions:
+# if chrom != last_chrom:
+# # Queue was not intronic (3'-terminal of last chrom)
+# for row in queue:
+# yield row
+# queue = []
+# last_chrom = chrom
+# last_gene = name if name != '-' else None
+# # Track genes; smooth over intronic '-' regions
+# if name == '-':
+# # Not sure if intronic -- enqueue
+# queue.append((chrom, start, end, name))
+# else:
+# if name == last_gene:
+# # Queued row(s) were intronic
+# if queue:
+# for cm, st, ed, _nm in queue:
+# yield cm, st, ed, name
+# queue = []
+# else:
+# # Queued row(s) were intergenic
+# for row in queue:
+# yield row
+# queue = []
+# last_gene = name
+# yield chrom, start, end, name
+# # Remainder
+# for row in queue:
+# yield row
+
+
+
+# _____________________________________________________________________________
+# Clean/shorten interval names
+
+def shorten_labels(interval_rows):
+ """Reduce multi-accession interval labels to the minimum consistent.
+
+ So: BED or interval_list files have a label for every region. We want this
+ to be a short, unique string, like the gene name. But if an interval list is
+ instead a series of accessions, including additional accessions for
+ sub-regions of the gene, we can extract a single accession that covers the
+ maximum number of consecutive regions that share this accession.
+
+ e.g.
+
+ ...
+ chr1 879125 879593 + mRNA|JX093079,ens|ENST00000342066,mRNA|JX093077,ref|SAMD11,mRNA|AF161376,mRNA|JX093104
+ chr1 880158 880279 + ens|ENST00000483767,mRNA|AF161376,ccds|CCDS3.1,ref|NOC2L
+ ...
+
+ becomes:
+
+ chr1 879125 879593 + mRNA|AF161376
+ chr1 880158 880279 + mRNA|AF161376
+ """
+ longest_name_len = 0
+ curr_names = set()
+ curr_gene_coords = []
+
+ for row in interval_rows:
+ next_coords = row[:-1]
+ next_names = set(row[-1].rstrip().split(','))
+ assert len(next_names)
+ overlap = curr_names.intersection(next_names)
+ if overlap:
+ # Continuing the same gene; update shared accessions
+ curr_names = filter_names(overlap)
+ curr_gene_coords.append(next_coords)
+ else:
+ # End of the old gene -- emit coords + shared name(s)
+ for coords in curr_gene_coords:
+ out_row = emit(coords, curr_names)
+ yield out_row
+ longest_name_len = max(longest_name_len, len(out_row[-1]))
+
+ # Start of a new gene
+ curr_gene_coords = [next_coords]
+ curr_names = next_names
+ # Final emission
+ for coords in curr_gene_coords:
+ out_row = emit(coords, curr_names)
+ yield out_row
+ longest_name_len = max(longest_name_len, len(out_row[-1]))
+
+ logging.info("Longest name length: %d", longest_name_len)
+
+
+def filter_names(names, exclude=('mRNA',)):
+ """Remove less-meaningful accessions from the given set."""
+ if len(names) > 1:
+ ok_names = set(n for n in names
+ if not any(n.startswith(ex) for ex in exclude))
+ if ok_names:
+ return ok_names
+ # Names are not filter-worthy; leave them as they are for now
+ return names
+
+
+def emit(coords, names):
+ """Try filtering names again. Format the row for yielding."""
+ try:
+ final_name = shortest_name(filter_names(names))
+ out_row = list(coords) + [final_name]
+ except ValueError:
+ raise ValueError(coords)
+ return out_row
+
+
+def shortest_name(names):
+ """Return the shortest trimmed name from the given set."""
+ name = min(names, key=len)
+ if len(name) > 2 and '|' in name[1:-1]:
+ # Split 'DB|accession' and extract the accession sans-DB
+ name = name.split('|')[-1]
+ return name
+
+
+# _____________________________________________________________________________
+# Split targets
+
+# ENH - fix overlapping bins at midpoint of overlap? try to equalize bin size?
+# - or just concatenate the bins & divide evenly?
+# - do this for adjacent bins too?
+# e.g. --balance-adjacent
+def split_targets(region_rows, avg_size):
+ """Split large tiled intervals into smaller, consecutive targets.
+
+ For each of the tiled regions:
+
+ - Divide into equal-size (tile_size/avg_size) portions
+ - Emit the (chrom, start, end) coords of each portion
+
+ Bin the regions according to avg_size.
+ """
+ prev_chrom = None
+ prev_start = None
+ prev_end = -1
+ for chrom, start, end, name in region_rows:
+ if (chrom, start, end) == (prev_chrom, prev_start, prev_end):
+ # Skip duplicate regions, even if names differ
+ # ENH - Update the name?
+ logging.info("Duplicate row %s %s:%d-%d", name, chrom, start, end)
+ continue
+ if chrom == prev_chrom and start <= prev_end:
+ if end > prev_end:
+ # Partial overlap: already handled the initial part of this bin
+ start = prev_end
+ else:
+ # Complete overlap: nothing to do on this bin
+ logging.info("Bin %s %s:%s-%s fully covered by previous bin"
+ "%s:%s-%s",
+ name, chrom, start, end,
+ prev_chrom, prev_start, prev_end)
+ continue
+ prev_chrom = chrom
+ prev_start = start
+ span = end - start
+ if span >= avg_size * 1.5:
+ logging.info("Splitting: %s %s",
+ name.ljust(15), str(end - start + 1).rjust(6))
+ # Divide the background region into equal-sized bins
+ nbins = round(span / avg_size) or 1
+ bin_size = span / nbins
+ # Locate & emit background bins
+ bin_start = start
+ while bin_start < (end - 2):
+ bin_end = bin_start + bin_size
+ yield (chrom, int(bin_start), int(bin_end), name)
+ prev_end = int(bin_end)
+ bin_start = bin_end
+ else:
+ yield (chrom, start, end, name)
+ prev_end = end
+
diff --git a/cnvlib/vary.py b/cnvlib/vary.py
new file mode 100644
index 0000000..932f8bc
--- /dev/null
+++ b/cnvlib/vary.py
@@ -0,0 +1,340 @@
+"""An array of genomic intervals, treated as variant loci."""
+from __future__ import absolute_import, division, print_function
+
+import logging
+
+import numpy as np
+import pandas as pd
+import vcf
+
+from . import gary
+
+
+class VariantArray(gary.GenomicArray):
+ """An array of genomic intervals, treated as variant loci.
+
+ Required columns: chromosome, start, end, ref, alt
+ """
+ _required_columns = ("chromosome", "start", "end", "ref", "alt")
+ _required_dtypes = ("string", "int", "int", "string", "string")
+ # Extra: somatic, zygosity, depth, alt_count, alt_freq
+
+ def __init__(self, data_table, meta_dict=None):
+ gary.GenomicArray.__init__(self, data_table, meta_dict)
+
+ def mirrored_baf(self, above_half=True, tumor_boost=True):
+ """Mirrored B-allele frequencies (BAFs).
+
+ Flip BAFs to be all above 0.5 (if `above_half`) or below 0.5, for
+ consistency.
+
+ With `tumor_boost`, normalize tumor-sample allele frequencies to the
+ matched normal allele frequencies.
+ """
+ if tumor_boost and "n_alt_freq" in self:
+ alt_freqs = self.tumor_boost()
+ else:
+ alt_freqs = self["alt_freq"]
+ fromhalf = np.abs(alt_freqs - 0.5)
+ if above_half:
+ return fromhalf + 0.5
+ else:
+ return 0.5 - fromhalf
+
+ def tumor_boost(self):
+ """TumorBoost normalization of tumor-sample allele frequencies.
+
+ De-noises the signal for detecting LOH.
+ """
+ if "n_alt_freq" in self:
+ n_freqs = self["n_alt_freq"]
+ else:
+ raise ValueError("TumorBoost requires a paired normal sample in "
+ "the VCF.")
+ t_freqs = self["alt_freq"]
+ return _tumor_boost(t_freqs, n_freqs)
+
+ # I/O
+
+ @classmethod
+ def read_vcf(cls, infile, sample_id=None, normal_id=None, min_depth=None,
+ skip_hom=False, skip_reject=False, skip_somatic=False):
+ """Parse SNV coordinates from a VCF file into a VariantArray."""
+ if isinstance(infile, basestring):
+ vcf_reader = vcf.Reader(filename=infile)
+ else:
+ vcf_reader = vcf.Reader(infile)
+ if not vcf_reader.samples:
+ logging.warn("VCF file %s has no samples; parsing minimal info",
+ infile)
+ return cls._read_vcf_nosample(infile, sample_id, skip_reject)
+
+ columns = [
+ "chromosome", "start", "end", "ref", "alt",
+ "somatic", "zygosity", "depth", "alt_count"]
+ sample_id, normal_id = _select_sample(vcf_reader, sample_id, normal_id)
+ if normal_id:
+ columns.extend(["n_zygosity", "n_depth", "n_alt_count"])
+ rows = _parse_records(vcf_reader, sample_id, normal_id, skip_reject)
+ table = pd.DataFrame.from_records(rows, columns=columns)
+ table["alt_freq"] = table["alt_count"] / table["depth"]
+ if normal_id:
+ table["n_alt_freq"] = table["n_alt_count"] / table["n_depth"]
+ # Filter out records as requested
+ cnt_depth = cnt_hom = cnt_som = 0
+ if min_depth:
+ dkey = "n_depth" if "n_depth" in table else "depth"
+ idx_depth = table[dkey] >= min_depth
+ cnt_depth = (~idx_depth).sum()
+ table = table[idx_depth]
+ if skip_hom:
+ zkey = "n_zygosity" if "n_zygosity" in table else "zygosity"
+ idx_het = (table[zkey] != 0.0) & (table[zkey] != 1.0)
+ cnt_hom = (~idx_het).sum()
+ table = table[idx_het]
+ if skip_somatic:
+ idx_som = table["somatic"]
+ cnt_som = idx_som.sum()
+ table = table[~idx_som]
+ logging.info("Skipped records: %d somatic, %d depth, %d homozygous",
+ cnt_som, cnt_depth, cnt_hom)
+
+ return cls(table, {"sample_id": sample_id})
+
+ @classmethod
+ def _read_vcf_nosample(cls, vcf_file, sample_id=None, skip_reject=False):
+ table = pd.read_table(vcf_file,
+ comment="#",
+ header=None,
+ na_filter=False,
+ names=["chromosome", "start", "_ID", "ref", "alt",
+ "_QUAL", "filter", "info"],
+ usecols=cls._required_columns,
+ # usecols=["chromosome", "start", "ref", "alt",
+ # # "filter", "info",
+ # ],
+ # ENH: converters=func -> to parse each col
+ dtype=dict(zip(cls._required_columns,
+ cls._required_dtypes)),
+ )
+ # ENH: do things with filter, info
+ # if skip_reject and record.FILTER and len(record.FILTER) > 0:
+ table['end'] = table['start'] # TODO: _get_end
+ table['start'] -= 1
+ table = table.loc[:, cls._required_columns]
+ return cls(table, {"sample_id": sample_id})
+
+
+ # def write_vcf(self, outfile=sys.stdout):
+ # """Write data to a file or handle in VCF format."""
+ # # grab from export.export_vcf()
+
+
+def _select_sample(vcf_reader, sample_id, normal_id):
+ """Select a sample ID in the VCF; ensure it's valid."""
+ peds = list(_parse_pedigrees(vcf_reader))
+ if sample_id is None and normal_id is None and peds:
+ # Use the VCF header to select the tumor sample
+ # Take the first entry, if any
+ sample_id, normal_id = peds[0]
+ elif sample_id and normal_id:
+ # Take the given IDs at face value, just verify below
+ pass
+ elif sample_id:
+ if peds:
+ for sid, nid in peds:
+ if sid == sample_id:
+ normal_id = nid
+ break
+ # if normal_id is None and len(vcf_reader.samples == 2):
+ # normal_id = next(s for s in vcf_reader.samples if s != sample_id)
+ elif normal_id:
+ if peds:
+ for sid, nid in peds:
+ if nid == normal_id:
+ sample_id = sid
+ break
+ else:
+ try:
+ sample_id = next(s for s in vcf_reader.samples if s != normal_id)
+ except StopIteration:
+ raise ValueError(
+ "No other sample in VCF besides the specified normal " +
+ normal_id + "; did you mean to use this as the sample_id "
+ "instead?")
+ else:
+ sample_id = vcf_reader.samples[0]
+
+ _confirm_unique(sample_id, vcf_reader.samples)
+ if normal_id:
+ _confirm_unique(normal_id, vcf_reader.samples)
+ logging.info("Selected test sample " + sample_id +
+ (" and control sample %s" % normal_id if normal_id else ''))
+ return sample_id, normal_id
+
+
+def _parse_pedigrees(vcf_reader):
+ """Extract tumor/normal pair sample IDs from the VCF header.
+
+ Return an iterable of (tumor sample ID, normal sample ID).
+ """
+ if "PEDIGREE" in vcf_reader.metadata:
+ for tag in vcf_reader.metadata["PEDIGREE"]:
+ if "Derived" in tag:
+ sample_id = tag["Derived"]
+ normal_id = tag["Original"]
+ logging.debug("Found tumor sample %s and normal sample %s "
+ "in the VCF header PEDIGREE tag",
+ sample_id, normal_id)
+ yield sample_id, normal_id
+
+ elif "GATKCommandLine" in vcf_reader.metadata:
+ for tag in vcf_reader.metadata["GATKCommandLine"]:
+ if tag.get("ID") == "MuTect": # any others OK?
+ options = dict(kv.split("=", 1)
+ for kv in tag["CommandLineOptions"].split()
+ if '=' in kv)
+ sample_id = options.get('tumor_sample_name')
+ normal_id = options['normal_sample_name']
+ logging.debug("Found tumor sample %s and normal sample "
+ "%s in the MuTect VCF header",
+ sample_id, normal_id)
+ yield sample_id, normal_id
+
+
+def _confirm_unique(sample_id, samples):
+ occurrences = [s for s in samples if s == sample_id]
+ if len(occurrences) != 1:
+ raise ValueError(
+ "Did not find a single sample ID '%s' in: %s"
+ % (sample_id, samples))
+
+
+def _parse_records(vcf_reader, sample_id, normal_id, skip_reject):
+ """Parse VCF records into DataFrame rows.
+
+ Apply filters to skip records with low depth, homozygosity, the REJECT
+ flag, or the SOMATIC info field.
+ """
+ cnt_reject = 0 # For logging
+ for record in vcf_reader:
+ is_som = False
+ if skip_reject and record.FILTER and len(record.FILTER) > 0:
+ cnt_reject += 1
+ continue
+ if record.INFO.get("SOMATIC"):
+ is_som = True
+
+ sample = record.genotype(sample_id)
+ depth, zygosity, alt_count = _extract_genotype(sample)
+ if normal_id:
+ normal = record.genotype(normal_id)
+ n_depth, n_zygosity, n_alt_count = _extract_genotype(normal)
+ if n_zygosity == 0:
+ is_som = True
+
+ # Split multiallelics?
+ # XXX Ensure sample genotypes are handled properly
+ for alt in record.ALT:
+ posn = record.POS - 1
+ end = _get_end(posn, alt, record.INFO)
+ row = (record.CHROM, posn, end, record.REF, str(alt),
+ is_som,
+ zygosity,
+ depth,
+ alt_count,
+ )
+ if normal_id:
+ row += (n_zygosity, n_depth, n_alt_count)
+ yield row
+
+ if cnt_reject:
+ logging.info('Filtered out %d records', cnt_reject)
+
+
+def _extract_genotype(sample):
+ if "DP" in sample.data._fields:
+ depth = sample.data.DP
+ else:
+ # SV, probably
+ depth = alt_count = None
+ if sample.is_het:
+ zygosity = 0.5
+ elif sample.gt_type == 0:
+ zygosity = 0.0
+ else:
+ zygosity = 1.0
+ alt_count = (_get_alt_count(sample) if sample.gt_type else 0.0)
+ return depth, zygosity, alt_count
+
+
+def _get_alt_count(sample):
+ """Get the alternative allele count from a sample in a VCF record."""
+ if "AD" in sample.data._fields and sample.data.AD is not None:
+ # GATK and other callers
+ if isinstance(sample.data.AD, (list, tuple)):
+ alt_count = float(sample.data.AD[1])
+ # VarScan
+ else:
+ alt_count = float(sample.data.AD)
+ elif "CLCAD2" in sample.data._fields and sample.data.CLCAD2 is not None:
+ # Qiagen CLC Genomics Server -- similar to GATK's AD
+ alt_count = float(sample.data.CLCAD2[1])
+ elif "AO" in sample.data._fields:
+ if sample.data.AO:
+ if isinstance(sample.data.AO, (list, tuple)):
+ alt_count = sum(map(float, sample.data.AO))
+ else:
+ alt_count = float(sample.data.AO)
+ else:
+ alt_count = 0.0
+ else:
+ logging.warn("Skipping %s:%d %s; "
+ "unsure how to get alternative allele count: %s",
+ sample.site.CHROM, sample.site.POS, sample.site.REF,
+ sample.data)
+ alt_count = None # or 0 or "missing data"?
+ return alt_count
+
+
+def _get_end(posn, alt, info):
+ """Get record end position."""
+ if "END" in info:
+ # Structural variant
+ return info['END']
+ return posn + len(alt)
+
+
+def _tumor_boost(t_freqs, n_freqs):
+ """Normalize tumor-sample allele frequencies.
+
+ boosted = { 0.5 (t/n) if t < n
+ 1 - 0.5(1-t)/(1-n) otherwise
+
+ See: TumorBoost, Bengtsson et al. 2010
+ """
+ lt_mask = (t_freqs < n_freqs)
+ lt_idx = np.nonzero(lt_mask)[0]
+ gt_idx = np.nonzero(~lt_mask)[0]
+ out = np.zeros_like(t_freqs)
+ out[lt_idx] = 0.5 * t_freqs.take(lt_idx) / n_freqs.take(lt_idx)
+ out[gt_idx] = 1 - 0.5 * (1 - t_freqs.take(gt_idx)
+ ) / (1 - n_freqs.take(gt_idx))
+ return out
+
+
+def _allele_specific_copy_numbers(segarr, varr, ploidy=2):
+ """Split total copy number between alleles based on BAF.
+
+ See: PSCBS, Bentsson et al. 2011
+ """
+ seg_depths = ploidy * np.exp2(segarr["log2"])
+ seg_bafs = np.asarray([(np.median(subvarr.mirrored_baf())
+ if len(subvarr) else np.nan)
+ for _seg, subvarr in varr.by_ranges(segarr)])
+ cn1 = 0.5 * (1 - seg_bafs) * seg_depths
+ cn2 = seg_depths - cn1
+ # segout = segarr.copy()
+ # segout.update({"baf": seg_bafs, "CN1": cn1, "CN2": cn2})
+ # return segout
+ return pd.DataFrame({"baf": seg_bafs, "CN1": cn1, "CN2"