[med-svn] [bedops] 01/05: New upstream version 2.4.26+dfsg
Olivier Sallou
osallou at debian.org
Wed Jun 21 12:27:23 UTC 2017
This is an automated email from the git hooks/post-receive script.
osallou pushed a commit to branch master
in repository bedops.
commit eabeaa5baec139a1a3f961561b33e75066f92b88
Author: Olivier Sallou <osallou at debian.org>
Date: Wed Jun 21 12:02:56 2017 +0000
New upstream version 2.4.26+dfsg
---
.gitignore | 63 +
.travis.yml | 34 +-
Makefile | 52 +-
README.md | 22 +-
applications/bed/bedextract/src/ExtractRows.cpp | 2 +-
applications/bed/bedmap/src/Bedmap.cpp | 38 +-
applications/bed/bedmap/src/Input.hpp | 25 +-
applications/bed/bedmap/src/TDefs.hpp | 23 +-
applications/bed/bedops/src/BedPadReader.hpp | 2 +-
applications/bed/bedops/src/Bedops.cpp | 30 +-
applications/bed/bedops/src/Input.hpp | 2 +-
applications/bed/bedops/src/Makefile.darwin | 2 +-
applications/bed/bedops/test/Regression.java | 2 +-
applications/bed/bedops/test/TestPlan.xml | 55 +
.../closestfeats/doc/Usage.Statement.Version1.2 | 45 -
applications/bed/closestfeats/src/BedReader.hpp | 2 +-
.../bed/closestfeats/src/ClosestFeature.cpp | 2 +-
applications/bed/closestfeats/src/Input.hpp | 2 +-
applications/bed/closestfeats/src/Printers.hpp | 2 +-
applications/bed/conversion/src/convert2bed.c | 2577 +++++++++++++++-----
applications/bed/conversion/src/convert2bed.h | 101 +-
.../src/wrappers/bam2bed_slurm} | 79 +-
.../src/wrappers/bam2starch_slurm} | 79 +-
applications/bed/sort-bed/src/CheckSort.cpp | 2 +-
applications/bed/sort-bed/src/Makefile | 14 +-
applications/bed/sort-bed/src/Makefile.darwin | 11 +-
applications/bed/sort-bed/src/Sort.cpp | 2 +-
applications/bed/sort-bed/src/SortDetails.cpp | 50 +-
applications/bed/sort-bed/src/Structures.hpp | 49 +-
.../src/update-sort-bed-migrate-candidates.py | 613 +++++
.../bed/sort-bed/src/update-sort-bed-slurm.py | 426 ++++
.../sort-bed/src/update-sort-bed-starch-slurm.py | 462 ++++
applications/bed/starch/src/Makefile | 16 +-
applications/bed/starch/src/Makefile.darwin | 25 +-
applications/bed/starch/src/starch-diff.py | 213 ++
applications/bed/starch/src/starch.c | 104 +-
applications/bed/starch/src/starch.h | 65 +-
applications/bed/starch/src/starchcat.c | 1415 +++++++----
applications/bed/starch/src/starchcat.h | 85 +-
.../bed/starch/src/starchcluster_gnuParallel.tcsh | 4 +-
applications/bed/starch/src/starchcluster_sge.tcsh | 4 +-
...chcluster_sge.tcsh => starchcluster_slurm.tcsh} | 37 +-
applications/bed/starch/src/starchstrip.c | 898 +++++++
applications/bed/starch/src/starchstrip.h | 143 ++
applications/bed/starch/src/unstarch.c | 285 ++-
applications/bed/starch/src/unstarch.h | 113 +-
docs/_static/custom.css | 12 +-
docs/conf.py | 8 +-
.../reference/file-management/compression.rst | 3 +
.../file-management/compression/starch-diff.rst | 52 +
.../compression/starch-specification.rst | 16 +-
.../file-management/compression/starch.rst | 61 +-
.../file-management/compression/starchcat.rst | 57 +-
.../file-management/compression/starchstrip.rst | 101 +
.../file-management/compression/unstarch.rst | 156 +-
.../file-management/conversion/gtf2bed.rst | 12 +-
.../conversion/parallel_bam2bed.rst | 12 +-
.../conversion/parallel_bam2starch.rst | 10 +-
.../file-management/conversion/psl2bed.rst | 2 +-
.../file-management/conversion/vcf2bed.rst | 2 +-
.../reference/file-management/sorting/sort-bed.rst | 22 +-
.../reference/set-operations/bedextract.rst | 2 +-
docs/content/reference/set-operations/bedops.rst | 2 +-
.../reference/set-operations/closest-features.rst | 2 +-
docs/content/reference/statistics/bedmap.rst | 23 +-
docs/content/release.rst | 8 +-
docs/content/revision-history.rst | 451 +++-
docs/content/summary.rst | 133 +-
docs/content/usage-examples/starchcluster.rst | 6 +-
docs/index.rst | 26 +-
.../general-headers/algorithm/WindowSweep.hpp | 6 +-
.../general-headers/algorithm/bed/FindBedRange.hpp | 2 +-
.../algorithm/visitors/BedVisitors.hpp | 3 +-
.../algorithm/visitors/NumericalVisitors.hpp | 2 +-
.../algorithm/visitors/OtherVisitors.hpp | 2 +-
.../algorithm/visitors/VisitorFactory.hpp | 2 +-
.../algorithm/visitors/Visitors.hpp | 2 +-
.../algorithm/visitors/bed/BedBaseVisitor.hpp | 4 +-
.../algorithm/visitors/bed/EchoMapBedVisitor.hpp | 2 +-
.../visitors/bed/EchoMapIntersectLengthVisitor.hpp | 5 +-
.../algorithm/visitors/bed/OvrAggregateVisitor.hpp | 2 +-
.../visitors/bed/OvrUniqueFractionVisitor.hpp | 2 +-
.../algorithm/visitors/bed/OvrUniqueVisitor.hpp | 2 +-
.../WeightedAverageVisitor.hpp} | 53 +-
.../algorithm/visitors/helpers/NamedVisitors.hpp | 65 +-
.../visitors/helpers/ProcessBedVisitorRow.hpp | 2 +-
.../visitors/helpers/ProcessVisitorRow.hpp | 2 +-
.../visitors/numerical/AverageVisitor.hpp | 2 +-
.../visitors/numerical/CoeffVariationVisitor.hpp | 2 +-
.../algorithm/visitors/numerical/CountVisitor.hpp | 2 +-
.../visitors/numerical/ExtremeVisitor.hpp | 69 +-
.../visitors/numerical/IndicatorVisitor.hpp | 2 +-
.../numerical/MedianAbsoluteDeviationVisitor.hpp | 2 +-
.../algorithm/visitors/numerical/MedianVisitor.hpp | 2 +-
.../numerical/RollingKthAverageVisitor.hpp | 2 +-
.../visitors/numerical/RollingKthVisitor.hpp | 2 +-
.../algorithm/visitors/numerical/StdevVisitor.hpp | 2 +-
.../algorithm/visitors/numerical/SumVisitor.hpp | 2 +-
.../visitors/numerical/TrimmedMeanVisitor.hpp | 2 +-
.../visitors/numerical/VarianceVisitor.hpp | 2 +-
.../algorithm/visitors/other/EchoVisitor.hpp | 2 +-
.../algorithm/visitors/other/MultiVisitor.hpp | 2 +-
.../data/bed/AllocateIterator_BED_starch.hpp | 5 +-
interfaces/general-headers/data/bed/Bed.hpp | 331 +--
.../general-headers/data/bed/BedCheckIterator.hpp | 78 +-
interfaces/general-headers/data/bed/BedCompare.hpp | 130 +-
.../general-headers/data/bed/BedDistances.hpp | 2 +-
interfaces/general-headers/data/bed/BedTypes.hpp | 9 +-
.../data/measurement/AssayMeasurement.hpp | 2 +-
.../general-headers/data/measurement/NaN.hpp | 2 +-
.../data/measurement/SelectMeasureType.hpp | 2 +-
.../general-headers/data/starch/starchApi.hpp | 55 +-
.../data/starch/starchBase64Coding.h | 2 +-
.../general-headers/data/starch/starchConstants.h | 2 +-
.../data/starch/starchFileHelpers.h | 20 +-
.../general-headers/data/starch/starchHelpers.h | 17 +-
.../data/starch/starchMetadataHelpers.h | 40 +-
.../general-headers/data/starch/starchSha1Digest.h | 6 +-
.../general-headers/data/starch/unstarchHelpers.h | 57 +-
.../general-headers/suite/BEDOPS.Constants.hpp | 30 +-
.../general-headers/suite/BEDOPS.Version.hpp | 4 +-
.../general-headers/utility/AllocateIterator.hpp | 2 +-
interfaces/general-headers/utility/Assertion.hpp | 2 +-
interfaces/general-headers/utility/ByLine.hpp | 2 +-
interfaces/general-headers/utility/Exception.hpp | 2 +-
interfaces/general-headers/utility/FPWrap.hpp | 2 +-
interfaces/general-headers/utility/Factory.hpp | 2 +-
interfaces/general-headers/utility/Formats.hpp | 2 +-
.../general-headers/utility/IteratorPair.hpp | 2 +-
.../general-headers/utility/OrderCompare.hpp | 2 +-
.../general-headers/utility/PooledMemory.hpp | 508 ++++
interfaces/general-headers/utility/PrintTypes.hpp | 2 +-
.../general-headers/utility/SingletonType.hpp | 2 +-
interfaces/general-headers/utility/Typify.hpp | 2 +-
interfaces/src/algorithm/sweep/WindowSweepImpl.cpp | 2 +-
.../algorithm/sweep/WindowSweepImpl.specialize.cpp | 2 +-
interfaces/src/data/measurement/NaN.cpp | 2 +-
interfaces/src/data/starch/starchBase64Coding.c | 2 +-
interfaces/src/data/starch/starchConstants.c | 2 +-
interfaces/src/data/starch/starchFileHelpers.c | 2 +-
interfaces/src/data/starch/starchHelpers.c | 442 +++-
interfaces/src/data/starch/starchMetadataHelpers.c | 316 ++-
interfaces/src/data/starch/unstarchHelpers.c | 434 +++-
packaging/deb/Dockerfile | 8 +-
packaging/deb/control | 2 +-
packaging/os_x/BEDOPS.pkgproj | 139 +-
packaging/rpm/Dockerfile | 12 +-
packaging/rpm/bedops.spec | 2 +-
system.mk/Makefile.darwin | 21 +
system.mk/Makefile.linux | 15 +
150 files changed, 9969 insertions(+), 2471 deletions(-)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..3398ce1
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,63 @@
+# -*- mode: gitignore; -*-
+*~
+\#*\#
+/.emacs.desktop
+/.emacs.desktop.lock
+*.elc
+auto-save-list
+tramp
+.\#*
+
+# Org-mode
+.org-id-locations
+*_archive
+
+# flymake-mode
+*_flymake.*
+
+# eshell files
+/eshell/history
+/eshell/lastdir
+
+# elpa packages
+/elpa/
+
+# reftex files
+*.rel
+
+# AUCTeX auto folder
+/auto/
+
+# Object files
+*.o
+*.ko
+*.obj
+*.elf
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Libraries
+*.lib
+*.a
+*.la
+*.lo
+
+# Shared objects (inc. Windows DLLs)
+*.dll
+*.so
+*.so.*
+*.dylib
+
+# Executables
+*.exe
+*.out
+*.app
+*.i*86
+*.x86_64
+*.hex
+
+# OS X stuff
+.DS_Store
+__MACOSX
diff --git a/.travis.yml b/.travis.yml
index ace9a97..ec53eeb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,29 +1,19 @@
language: cpp
os:
- linux
+ - osx
compiler:
- gcc
- # - clang
-# Change this to your needs
+ - clang
before_install:
- # g++ 4.8.3
- - if [ "$CXX" == "g++" ]; then sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; fi
- # clang 3.4
- - if [ "$CXX" == "clang++" ]; then sudo add-apt-repository -y ppa:h-rayflood/llvm; fi
- - sudo apt-get update -qq
+ - if [[ "$CC" == "gcc" ]]; then if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; fi; fi
+ - if [[ "$CXX" == "g++" ]]; then if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; fi; fi
+ - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get update -qq; fi
install:
- # g++ 4.8.3
- - if [ "$CXX" == "g++" ]; then sudo apt-get install -qq g++-4.8; fi
- - if [ "$CXX" == "g++" ]; then sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 50; fi
- # clang 3.4
- - if [ "$CXX" == "clang++" ]; then sudo apt-get install --allow-unauthenticated -qq clang-3.4; fi
- - if [ "$CXX" == "clang++" ]; then export CXXFLAGS="-std=c++0x -stdlib=libc++"; fi
- - if [ "$CXX" == "clang++" ]; then svn co --quiet http://llvm.org/svn/llvm-project/libcxx/trunk libcxx; fi
- - if [ "$CXX" == "clang++" ]; then cd libcxx/lib && bash buildit; fi
- - if [ "$CXX" == "clang++" ]; then sudo cp ./libc++.so.1.0 /usr/lib/; fi
- - if [ "$CXX" == "clang++" ]; then sudo mkdir /usr/include/c++/v1; fi
- - if [ "$CXX" == "clang++" ]; then cd .. && sudo cp -r include/* /usr/include/c++/v1/; fi
- - if [ "$CXX" == "clang++" ]; then cd /usr/lib && sudo ln -sf libc++.so.1.0 libc++.so; fi
- - if [ "$CXX" == "clang++" ]; then sudo ln -sf libc++.so.1.0 libc++.so.1 && cd $cwd; fi
- #- if [ "$CXX" == "clang++" ]; then export CXX="clang++-3.4"; fi
-script: make -j4
\ No newline at end of file
+ - if [[ "$CC" == "gcc" ]]; then if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install -qq gcc-4.8; fi; fi
+ - if [[ "$CC" == "gcc" ]]; then if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 50; fi; fi
+ - if [[ "$CXX" == "g++" ]]; then if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install -qq g++-4.8; fi; fi
+ - if [[ "$CXX" == "g++" ]]; then if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 50; fi; fi
+script:
+ - if [[ "$CC" == "gcc" ]]; then if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make -j4; fi; fi
+ - if [[ "$CC" == "clang" ]]; then if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then make -j4; fi; fi
\ No newline at end of file
diff --git a/Makefile b/Makefile
index d1bd96b..cc0d879 100644
--- a/Makefile
+++ b/Makefile
@@ -22,8 +22,11 @@ debug: default
gprof: default
-install: prep_c install_conversion_scripts install_starchcluster_scripts
+install: prep_c install_conversion_scripts install_starch_scripts
-cp ${APPDIR}/sort-bed/bin/sort-bed ${BINDIR}/
+ -cp ${APPDIR}/sort-bed/bin/update-sort-bed-slurm ${BINDIR}/
+ -cp ${APPDIR}/sort-bed/bin/update-sort-bed-starch-slurm ${BINDIR}/
+ -cp ${APPDIR}/sort-bed/bin/update-sort-bed-migrate-candidates ${BINDIR}/
-cp ${APPDIR}/bedops/bin/bedops ${BINDIR}/
-cp ${APPDIR}/closestfeats/bin/closest-features ${BINDIR}/
-cp ${APPDIR}/bedmap/bin/bedmap ${BINDIR}/
@@ -31,6 +34,7 @@ install: prep_c install_conversion_scripts install_starchcluster_scripts
-cp ${APPDIR}/starch/bin/starch ${BINDIR}/
-cp ${APPDIR}/starch/bin/unstarch ${BINDIR}/
-cp ${APPDIR}/starch/bin/starchcat ${BINDIR}/
+ -cp ${APPDIR}/starch/bin/starchstrip ${BINDIR}/
-cp ${APPDIR}/conversion/bin/convert2bed ${BINDIR}/
@@ -42,8 +46,11 @@ install: prep_c install_conversion_scripts install_starchcluster_scripts
prep_c:
mkdir -p ${BINDIR}
-install_debug: prep_c install_conversion_scripts install_starchcluster_scripts
+install_debug: prep_c install_conversion_scripts install_starch_scripts
-cp ${APPDIR}/sort-bed/bin/debug.sort-bed ${BINDIR}/
+ -cp ${APPDIR}/sort-bed/bin/update-sort-bed-slurm ${BINDIR}/
+ -cp ${APPDIR}/sort-bed/bin/update-sort-bed-starch-slurm ${BINDIR}/
+ -cp ${APPDIR}/sort-bed/bin/update-sort-bed-migrate-candidates ${BINDIR}/
-cp ${APPDIR}/bedops/bin/debug.bedops ${BINDIR}/
-cp ${APPDIR}/closestfeats/bin/debug.closest-features ${BINDIR}/
-cp ${APPDIR}/bedmap/bin/debug.bedmap ${BINDIR}/
@@ -51,10 +58,14 @@ install_debug: prep_c install_conversion_scripts install_starchcluster_scripts
-cp ${APPDIR}/starch/bin/debug.starch ${BINDIR}/
-cp ${APPDIR}/starch/bin/debug.unstarch ${BINDIR}/
-cp ${APPDIR}/starch/bin/debug.starchcat ${BINDIR}/
+ -cp ${APPDIR}/starch/bin/debug.starchstrip ${BINDIR}/
-cp ${APPDIR}/conversion/bin/debug.convert2bed ${BINDIR}/
-install_gprof: prep_c install_conversion_scripts install_starchcluster_scripts
+install_gprof: prep_c install_conversion_scripts install_starch_scripts
-cp ${APPDIR}/sort-bed/bin/gprof.sort-bed ${BINDIR}/
+ -cp ${APPDIR}/sort-bed/bin/update-sort-bed-slurm ${BINDIR}/
+ -cp ${APPDIR}/sort-bed/bin/update-sort-bed-starch-slurm ${BINDIR}/
+ -cp ${APPDIR}/sort-bed/bin/update-sort-bed-migrate-candidates ${BINDIR}/
-cp ${APPDIR}/bedops/bin/gprof.bedops ${BINDIR}/
-cp ${APPDIR}/closestfeats/bin/gprof.closest-features ${BINDIR}/
-cp ${APPDIR}/bedmap/bin/gprof.bedmap ${BINDIR}/
@@ -62,11 +73,14 @@ install_gprof: prep_c install_conversion_scripts install_starchcluster_scripts
-cp ${APPDIR}/starch/bin/gprof.starch ${BINDIR}/
-cp ${APPDIR}/starch/bin/gprof.unstarch ${BINDIR}/
-cp ${APPDIR}/starch/bin/gprof.starchcat ${BINDIR}/
+ -cp ${APPDIR}/starch/bin/gprof.starchstrip ${BINDIR}/
-cp ${APPDIR}/conversion/bin/gprof.convert2bed ${BINDIR}/
-install_starchcluster_scripts: prep_c
+install_starch_scripts: prep_c
-cp ${APPDIR}/starch/bin/starchcluster_sge ${BINDIR}/
-cp ${APPDIR}/starch/bin/starchcluster_gnuParallel ${BINDIR}/
+ -cp ${APPDIR}/starch/bin/starchcluster_slurm ${BINDIR}/
+ -cp ${APPDIR}/starch/bin/starch-diff ${BINDIR}/
install_conversion_scripts: prep_c
-cp ${APPDIR}/conversion/src/wrappers/bam2bed ${BINDIR}/bam2bed
@@ -88,13 +102,18 @@ install_conversion_scripts: prep_c
-cp ${APPDIR}/conversion/src/wrappers/vcf2starch ${BINDIR}/vcf2starch
-cp ${APPDIR}/conversion/src/wrappers/wig2starch ${BINDIR}/wig2starch
-cp ${APPDIR}/conversion/src/wrappers/bam2bed_sge ${BINDIR}/bam2bed_sge
+ -cp ${APPDIR}/conversion/src/wrappers/bam2bed_slurm ${BINDIR}/bam2bed_slurm
-cp ${APPDIR}/conversion/src/wrappers/bam2bed_gnuParallel ${BINDIR}/bam2bed_gnuParallel
-cp ${APPDIR}/conversion/src/wrappers/bam2starch_sge ${BINDIR}/bam2starch_sge
+ -cp ${APPDIR}/conversion/src/wrappers/bam2starch_slurm ${BINDIR}/bam2starch_slurm
-cp ${APPDIR}/conversion/src/wrappers/bam2starch_gnuParallel ${BINDIR}/bam2starch_gnuParallel
install_osx_packaging_bins: prep_c
mkdir -p ${OSXPKGDIR}
-cp ${APPDIR}/sort-bed/bin/sort-bed ${OSXPKGDIR}/
+ -cp ${APPDIR}/sort-bed/bin/update-sort-bed-slurm ${OSXPKGDIR}/
+ -cp ${APPDIR}/sort-bed/bin/update-sort-bed-starch-slurm ${OSXPKGDIR}/
+ -cp ${APPDIR}/sort-bed/bin/update-sort-bed-migrate-candidates ${OSXPKGDIR}/
-cp ${APPDIR}/bedops/bin/bedops ${OSXPKGDIR}/
-cp ${APPDIR}/closestfeats/bin/closest-features ${OSXPKGDIR}/
-cp ${APPDIR}/bedmap/bin/bedmap ${OSXPKGDIR}/
@@ -102,8 +121,11 @@ install_osx_packaging_bins: prep_c
-cp ${APPDIR}/starch/bin/starch ${OSXPKGDIR}/
-cp ${APPDIR}/starch/bin/unstarch ${OSXPKGDIR}/
-cp ${APPDIR}/starch/bin/starchcat ${OSXPKGDIR}/
+ -cp ${APPDIR}/starch/bin/starchstrip ${OSXPKGDIR}/
-cp ${APPDIR}/starch/bin/starchcluster_sge ${OSXPKGDIR}/starchcluster_sge
-cp ${APPDIR}/starch/bin/starchcluster_gnuParallel ${OSXPKGDIR}/starchcluster_gnuParallel
+ -cp ${APPDIR}/starch/bin/starchcluster_slurm ${OSXPKGDIR}/starchcluster_slurm
+ -cp ${APPDIR}/starch/bin/starch-diff ${OSXPKGDIR}/starch-diff
-cp ${APPDIR}/conversion/bin/convert2bed ${OSXPKGDIR}/
-cp ${APPDIR}/conversion/src/wrappers/bam2bed ${OSXPKGDIR}/bam2bed
-cp ${APPDIR}/conversion/src/wrappers/gff2bed ${OSXPKGDIR}/gff2bed
@@ -124,17 +146,19 @@ install_osx_packaging_bins: prep_c
-cp ${APPDIR}/conversion/src/wrappers/vcf2starch ${OSXPKGDIR}/vcf2starch
-cp ${APPDIR}/conversion/src/wrappers/wig2starch ${OSXPKGDIR}/wig2starch
-cp ${APPDIR}/conversion/src/wrappers/bam2bed_sge ${OSXPKGDIR}/bam2bed_sge
+ -cp ${APPDIR}/conversion/src/wrappers/bam2bed_slurm ${OSXPKGDIR}/bam2bed_slurm
-cp ${APPDIR}/conversion/src/wrappers/bam2bed_gnuParallel ${OSXPKGDIR}/bam2bed_gnuParallel
-cp ${APPDIR}/conversion/src/wrappers/bam2starch_sge ${OSXPKGDIR}/bam2starch_sge
+ -cp ${APPDIR}/conversion/src/wrappers/bam2starch_slurm ${OSXPKGDIR}/bam2starch_slurm
-cp ${APPDIR}/conversion/src/wrappers/bam2starch_gnuParallel ${OSXPKGDIR}/bam2starch_gnuParallel
mkdir -p ${OSXLIBDIR}
update_bedops_version:
ifndef OLD_VER
- $(error Old version variable OLD_VER is undefined (e.g., 2.4.16))
+ $(error Old version variable OLD_VER is undefined (e.g., 2.4.25))
endif
ifndef NEW_VER
- $(error New version variable NEW_VER is undefined (e.g., 2.4.17))
+ $(error New version variable NEW_VER is undefined (e.g., 2.4.26))
endif
ifeq ($(KERNEL), Darwin)
sed -i "" -e "s/"$$OLD_VER"/"$$NEW_VER"/g" README.md
@@ -147,6 +171,14 @@ ifeq ($(KERNEL), Darwin)
sed -i "" -e "s/"$$OLD_VER"/"$$NEW_VER"/g" docs/index.rst
sed -i "" -e "s/"$$OLD_VER"/"$$NEW_VER"/g" docs/conf.py
sed -i "" -e "s/"$$OLD_VER"/"$$NEW_VER"/g" applications/bed/conversion/src/convert2bed.h
+ sed -i "" -e "s/"$$OLD_VER"/"$$NEW_VER"/g" applications/bed/starch/src/starchcluster_gnuParallel.tcsh
+ sed -i "" -e "s/"$$OLD_VER"/"$$NEW_VER"/g" applications/bed/starch/src/starchcluster_sge.tcsh
+ sed -i "" -e "s/"$$OLD_VER"/"$$NEW_VER"/g" applications/bed/starch/src/starchcluster_slurm.tcsh
+ sed -i "" -e "s/"$$OLD_VER"/"$$NEW_VER"/g" applications/bed/starch/src/starch-diff.py
+ sed -i "" -e "s/"$$OLD_VER"/"$$NEW_VER"/g" applications/bed/starch/src/Makefile.darwin
+ sed -i "" -e "s/"$$OLD_VER"/"$$NEW_VER"/g" applications/bed/sort-bed/src/update-sort-bed-slurm.py
+ sed -i "" -e "s/"$$OLD_VER"/"$$NEW_VER"/g" applications/bed/sort-bed/src/update-sort-bed-starch-slurm.py
+ sed -i "" -e "s/"$$OLD_VER"/"$$NEW_VER"/g" applications/bed/sort-bed/src/update-sort-bed-migrate-candidates.py
find docs/content -type f -exec sed -i "" -e "s/"$$OLD_VER"/"$$NEW_VER"/g" {} +
else
sed -i "s/"$$OLD_VER"/"$$NEW_VER"/g" README.md
@@ -159,6 +191,14 @@ else
sed -i "s/"$$OLD_VER"/"$$NEW_VER"/g" docs/index.rst
sed -i "s/"$$OLD_VER"/"$$NEW_VER"/g" docs/conf.py
sed -i "s/"$$OLD_VER"/"$$NEW_VER"/g" applications/bed/conversion/src/convert2bed.h
+ sed -i "s/"$$OLD_VER"/"$$NEW_VER"/g" applications/bed/starch/src/starchcluster_gnuParallel.tcsh
+ sed -i "s/"$$OLD_VER"/"$$NEW_VER"/g" applications/bed/starch/src/starchcluster_sge.tcsh
+ sed -i "s/"$$OLD_VER"/"$$NEW_VER"/g" applications/bed/starch/src/starchcluster_slurm.tcsh
+ sed -i "s/"$$OLD_VER"/"$$NEW_VER"/g" applications/bed/starch/src/starch-diff.py
+ sed -i "s/"$$OLD_VER"/"$$NEW_VER"/g" applications/bed/starch/src/Makefile.darwin
+ sed -i "s/"$$OLD_VER"/"$$NEW_VER"/g" applications/bed/sort-bed/src/update-sort-bed-slurm.py
+ sed -i "s/"$$OLD_VER"/"$$NEW_VER"/g" applications/bed/sort-bed/src/update-sort-bed-starch-slurm.py
+ sed -i "s/"$$OLD_VER"/"$$NEW_VER"/g" applications/bed/sort-bed/src/update-sort-bed-migrate-candidates.py
find docs/content -type f -exec sed -i "s/"$$OLD_VER"/"$$NEW_VER"/g" {} +
endif
diff --git a/README.md b/README.md
index 7b4801c..3197176 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-## BEDOPS v2.4.20: high-performance genomic feature operations ##
+## BEDOPS v2.4.26: high-performance genomic feature operations ##
[](https://travis-ci.org/bedops/bedops) [](https://github.com/bedops/bedops/stargazers) [](https://github.com/bedops/bedops/blob/master/LICENSE) [](https://readthedocs.org/projects/bedops/?badge=latest)
@@ -6,9 +6,9 @@
### About ###
-**BEDOPS v2.4.20** is a suite of tools to address common questions raised in genomic studies — mostly with regard to overlap and proximity relationships between data sets. It aims to be scalable and flexible, facilitating the efficient and accurate analysis and management of large-scale genomic data.
+**BEDOPS v2.4.26** is a suite of tools to address common questions raised in genomic studies — mostly with regard to overlap and proximity relationships between data sets. It aims to be scalable and flexible, facilitating the efficient and accurate analysis and management of large-scale genomic data.
-The <a href="https://bedops.readthedocs.io/en/latest/content/overview.html#overview">overview</a> section of the **BEDOPS v2.4.20** documentation summarizes the toolkit, functionality and performance enhancements. The <a href="https://bedops.readthedocs.io/en/latest/index.html#reference">reference</a> table offers documentation for all applications and scripts.
+The <a href="https://bedops.readthedocs.io/en/latest/content/overview.html#overview">overview</a> section of the **BEDOPS v2.4.26** documentation summarizes the toolkit, functionality and performance enhancements. The <a href="https://bedops.readthedocs.io/en/latest/index.html#reference">reference</a> table offers documentation for all applications and scripts.
### Downloads ###
@@ -28,21 +28,21 @@ The <a href="https://bedops.readthedocs.io/en/latest/content/overview.html#overv
<tr>
<td valign="top">
<ul style="list-style-type:square; font-size:smaller; margin-left:0; margin-right:0px; padding-right:0px; padding-left:20px;">
-<li><a href="https://github.com/bedops/bedops/releases/download/v2.4.20/bedops_linux_x86_64-v2.4.20.tar.bz2">x86-64 (64-bit)</a> binaries</li>
-<li><a href="https://github.com/bedops/bedops/releases/download/v2.4.20/bedops_linux_i386-v2.4.20.tar.bz2">i386 (32-bit)</a> binaries</li>
+<li><a href="https://github.com/bedops/bedops/releases/download/v2.4.26/bedops_linux_x86_64-v2.4.26.tar.bz2">x86-64 (64-bit)</a> binaries</li>
+<li><a href="https://github.com/bedops/bedops/releases/download/v2.4.26/bedops_linux_i386-v2.4.26.tar.bz2">i386 (32-bit)</a> binaries</li>
<li><a href="http://bedops.readthedocs.io/en/latest/content/installation.html#linux">Installation instructions</a> for Linux hosts</li>
</ul>
</td>
<td valign="top">
<ul style="list-style-type:square; font-size:smaller; margin-left:0; margin-right:0px; padding-right:0px; padding-left:20px;">
-<li><a href="https://github.com/bedops/bedops/releases/download/v2.4.20/BEDOPS.2.4.20.pkg.zip">Intel (32-/64-bit, 10.7-10.11)</a> installer package</li>
+<li><a href="https://github.com/bedops/bedops/releases/download/v2.4.26/BEDOPS.2.4.26.pkg.zip">Intel (32-/64-bit, 10.7-10.12)</a> installer package</li>
<li><a href="http://bedops.readthedocs.io/en/latest/content/installation.html#mac-os-x">Installation instructions</a> for Mac OS X hosts</li>
</ul>
</td>
<td valign="top">
<ul style="list-style-type:square; font-size:smaller; margin-left:0; margin-right:0px; padding-right:0px; padding-left:20px;">
-<li><a href="https://github.com/bedops/bedops/archive/v2.4.20.tar.gz">Source code</a> (tar.gz)</li>
-<li><a href="https://github.com/bedops/bedops/archive/v2.4.20.zip">Source code</a> (zip)</li>
+<li><a href="https://github.com/bedops/bedops/archive/v2.4.26.tar.gz">Source code</a> (tar.gz)</li>
+<li><a href="https://github.com/bedops/bedops/archive/v2.4.26.zip">Source code</a> (zip)</li>
<li><a href="http://bedops.readthedocs.io/en/latest/content/installation.html#installation-via-source-code">Compilation instructions</a></li>
</ul>
</td>
@@ -51,11 +51,11 @@ The <a href="https://bedops.readthedocs.io/en/latest/content/overview.html#overv
### Documentation ###
-Complete documentation for **BEDOPS v2.4.20** tools is available at: <a href="https://bedops.readthedocs.io/en/latest/index.html">https://bedops.readthedocs.io/</a>
+Complete documentation for **BEDOPS v2.4.26** tools is available at: <a href="https://bedops.readthedocs.io/en/latest/index.html">https://bedops.readthedocs.io/</a>
### Citation ###
-If you use **BEDOPS v2.4.20** in your research, please cite the following manuscript:
+If you use **BEDOPS v2.4.26** in your research, please cite the following manuscript:
> Shane Neph, M. Scott Kuehn, Alex P. Reynolds, et al.
> [**BEDOPS: high-performance genomic feature operations**
@@ -63,7 +63,7 @@ If you use **BEDOPS v2.4.20** in your research, please cite the following manusc
### Copyright ###
-> Copyright (C) 2011-2016 Shane J. Neph, M. Scott Kuehn and Alex P. Reynolds
+> Copyright (C) 2011-2017 Shane J. Neph, M. Scott Kuehn and Alex P. Reynolds
>
> Source code, documentation and media assets released under <a href="https://github.com/bedops/bedops/blob/master/LICENSE">GNU Public License Version 2</a> (GPL v2).
diff --git a/applications/bed/bedextract/src/ExtractRows.cpp b/applications/bed/bedextract/src/ExtractRows.cpp
index c647566..7d9456f 100644
--- a/applications/bed/bedextract/src/ExtractRows.cpp
+++ b/applications/bed/bedextract/src/ExtractRows.cpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/applications/bed/bedmap/src/Bedmap.cpp b/applications/bed/bedmap/src/Bedmap.cpp
index d8b6385..4692a33 100644
--- a/applications/bed/bedmap/src/Bedmap.cpp
+++ b/applications/bed/bedmap/src/Bedmap.cpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -383,11 +383,15 @@ namespace BedMap {
typedef typename PT::PType PType;
rtn = new typename VTypes::EchoMapAll(PT(PType(), multivalColSep));
}
- else if ( nm == visName<typename VTypes::EchoMapLength>() )
- rtn = new typename VTypes::EchoMapLength;
- else if ( nm == visName<typename VTypes::EchoMapIntersectLength>() )
- rtn = new typename VTypes::EchoMapIntersectLength;
- else if ( nm == visName<typename VTypes::EchoMapRange>() )
+ else if ( nm == visName<typename VTypes::EchoMapLength>() ) {
+ typedef typename VTypes::EchoMapLength::ProcessType PT;
+ typedef typename PT::PType PType;
+ rtn = new typename VTypes::EchoMapLength(PT(PType(), multivalColSep));
+ } else if ( nm == visName<typename VTypes::EchoMapIntersectLength>() ) {
+ typedef typename VTypes::EchoMapIntersectLength::ProcessType PT;
+ typedef typename PT::PType PType;
+ rtn = new typename VTypes::EchoMapIntersectLength(PT(PType(), multivalColSep));
+ } else if ( nm == visName<typename VTypes::EchoMapRange>() )
rtn = new typename VTypes::EchoMapRange;
else if ( nm == visName<typename VTypes::EchoRefAll>() )
rtn = new typename VTypes::EchoRefAll;
@@ -497,9 +501,13 @@ namespace BedMap {
}
else if ( nm == visName<typename VTypes::Max>() )
rtn = new typename VTypes::Max(pt);
- else if ( nm == visName<typename VTypes::MaxElement>() ) {
- typedef typename VTypes::MaxElement::ProcessType MPT;
- rtn = new typename VTypes::MaxElement(MPT(precision, useScientific));
+ else if ( nm == visName<typename VTypes::MaxElementStable>() ) {
+ typedef typename VTypes::MaxElementStable::ProcessType MPT;
+ rtn = new typename VTypes::MaxElementStable(MPT(precision, useScientific));
+ }
+ else if ( nm == visName<typename VTypes::MaxElementRand>() ) {
+ typedef typename VTypes::MaxElementRand::ProcessType MPT;
+ rtn = new typename VTypes::MaxElementRand(MPT(precision, useScientific));
}
else if ( nm == visName<typename VTypes::Median>() )
rtn = new typename VTypes::Median(pt);
@@ -516,9 +524,13 @@ namespace BedMap {
}
else if ( nm == visName<typename VTypes::Min>() )
rtn = new typename VTypes::Min(pt);
- else if ( nm == visName<typename VTypes::MinElement>() ) {
- typedef typename VTypes::MinElement::ProcessType MPT;
- rtn = new typename VTypes::MinElement(MPT(precision, useScientific));
+ else if ( nm == visName<typename VTypes::MinElementRand>() ) {
+ typedef typename VTypes::MinElementRand::ProcessType MPT;
+ rtn = new typename VTypes::MinElementRand(MPT(precision, useScientific));
+ }
+ else if ( nm == visName<typename VTypes::MinElementStable>() ) {
+ typedef typename VTypes::MinElementStable::ProcessType MPT;
+ rtn = new typename VTypes::MinElementStable(MPT(precision, useScientific));
}
else if ( nm == visName<typename VTypes::StdDev>() )
rtn = new typename VTypes::StdDev(pt);
@@ -534,6 +546,8 @@ namespace BedMap {
}
else if ( nm == visName<typename VTypes::Variance>() )
rtn = new typename VTypes::Variance(pt);
+ else if ( nm == visName<typename VTypes::WMean>() )
+ rtn = new typename VTypes::WMean(pt);
return(rtn);
}
diff --git a/applications/bed/bedmap/src/Input.hpp b/applications/bed/bedmap/src/Input.hpp
index 258d5a0..9432798 100644
--- a/applications/bed/bedmap/src/Input.hpp
+++ b/applications/bed/bedmap/src/Input.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -34,6 +34,7 @@
#include <string>
#include <vector>
+#include "data/bed/BedCompare.hpp"
#include "algorithm/visitors/BedVisitors.hpp"
#include "algorithm/visitors/helpers/NamedVisitors.hpp"
#include "utility/Assertion.hpp"
@@ -247,12 +248,16 @@ namespace BedMap {
hasVisitor = addNoArgVisitor(Ext::Type2Type<typename VT::Indicator>());
else if ( next == details::name<typename VT::Max>() )
hasVisitor = addNoArgVisitor(Ext::Type2Type<typename VT::Max>());
- else if ( next == details::name<typename VT::MaxElement>() )
- hasVisitor = addNoArgVisitor(Ext::Type2Type<typename VT::MaxElement>());
+ else if ( next == details::name<typename VT::MaxElementRand>() )
+ hasVisitor = addNoArgVisitor(Ext::Type2Type<typename VT::MaxElementRand>());
+ else if ( next == details::name<typename VT::MaxElementStable>() )
+ hasVisitor = addNoArgVisitor(Ext::Type2Type<typename VT::MaxElementStable>());
else if ( next == details::name<typename VT::Min>() )
hasVisitor = addNoArgVisitor(Ext::Type2Type<typename VT::Min>());
- else if ( next == details::name<typename VT::MinElement>() )
- hasVisitor = addNoArgVisitor(Ext::Type2Type<typename VT::MinElement>());
+ else if ( next == details::name<typename VT::MinElementRand>() )
+ hasVisitor = addNoArgVisitor(Ext::Type2Type<typename VT::MinElementRand>());
+ else if ( next == details::name<typename VT::MinElementStable>() )
+ hasVisitor = addNoArgVisitor(Ext::Type2Type<typename VT::MinElementStable>());
else if ( next == details::name<typename VT::Average>() )
hasVisitor = addNoArgVisitor(Ext::Type2Type<typename VT::Average>());
else if ( next == details::name<typename VT::Variance>() )
@@ -263,6 +268,8 @@ namespace BedMap {
hasVisitor = addNoArgVisitor(Ext::Type2Type<typename VT::CoeffVariation>());
else if ( next == details::name<typename VT::Sum>() )
hasVisitor = addNoArgVisitor(Ext::Type2Type<typename VT::Sum>());
+ else if ( next == details::name<typename VT::WMean>() )
+ hasVisitor = addNoArgVisitor(Ext::Type2Type<typename VT::WMean>());
else if ( next == details::name<typename VT::Median>() )
hasVisitor = addNoArgVisitor(Ext::Type2Type<typename VT::Median>());
else if ( next == details::name<typename VT::MedianAbsoluteDeviation>() ) {
@@ -500,17 +507,21 @@ namespace BedMap {
usage << " --" + details::name<VT::MedianAbsoluteDeviation>() + " <mult=1> The median absolute deviation of overlapping elements in <map-file>.\n";
usage << " Multiply mad score by <mult>. 0 < mult, and mult is 1 by default.\n";
usage << " --" + details::name<VT::Max>() + " The highest score from overlapping elements in <map-file>.\n";
- usage << " --" + details::name<VT::MaxElement>() + " An element with the highest score from overlapping elements in <map-file>.\n";
+ usage << " --" + details::name<VT::MaxElementStable>() + " A (non-random) highest-scoring and overlapping element in <map-file>.\n";
+ usage << " --" + details::name<VT::MaxElementRand>() + " A random highest-scoring and overlapping element in <map-file>.\n";
usage << " --" + details::name<VT::Average>() + " The average score from overlapping elements in <map-file>.\n";
usage << " --" + details::name<VT::Median>() + " The median score from overlapping elements in <map-file>.\n";
usage << " --" + details::name<VT::Min>() + " The lowest score from overlapping elements in <map-file>.\n";
- usage << " --" + details::name<VT::MinElement>() + " An element with the lowest score from overlapping elements in <map-file>.\n";
+ usage << " --" + details::name<VT::MinElementStable>() + " A (non-random) lowest-scoring and overlapping element in <map-file>.\n";
+ usage << " --" + details::name<VT::MinElementRand>() + " A random lowest-scoring and overlapping element in <map-file>.\n";
usage << " --" + details::name<VT::StdDev>() + " The square root of the result of --" + details::name<VT::Variance>() + ".\n";
usage << " --" + details::name<VT::Sum>() + " Accumulated scores from overlapping elements in <map-file>.\n";
usage << " --" + details::name<VT::TMean>() + " <low> <hi> The mean score from overlapping elements in <map-file>, after\n";
usage << " ignoring the bottom <low> and top <hi> fractions of those scores.\n";
usage << " 0 <= low <= 1. 0 <= hi <= 1. low+hi <= 1.\n";
usage << " --" + details::name<VT::Variance>() + " The variance of scores from overlapping elements in <map-file>.\n";
+ usage << " --" + details::name<VT::WMean>() + " Weighted mean, scaled in proportion to the coverage of the <ref-file>\n";
+ usage << " element by each overlapping <map-file> element.\n";
usage << " \n";
usage << " ----------\n";
usage << " NON-SCORE:\n";
diff --git a/applications/bed/bedmap/src/TDefs.hpp b/applications/bed/bedmap/src/TDefs.hpp
index e26c7a3..b7c8202 100644
--- a/applications/bed/bedmap/src/TDefs.hpp
+++ b/applications/bed/bedmap/src/TDefs.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -27,7 +27,9 @@
#include "algorithm/visitors/BedVisitors.hpp"
#include "algorithm/visitors/helpers/ProcessBedVisitorRow.hpp"
#include "algorithm/visitors/helpers/ProcessVisitorRow.hpp"
+#include "data/bed/BedCompare.hpp"
#include "utility/Exception.hpp"
+#include "utility/OrderCompare.hpp"
namespace BedMap {
@@ -58,25 +60,32 @@ namespace BedMap {
typedef typename BaseClass::RefType RefType;
typedef typename BaseClass::MapType MapType;
- typedef Ordering::CompValueThenAddressGreater<MapType, MapType> MaxOrder;
- typedef Ordering::CompValueThenAddressLesser<MapType, MapType> MinOrder;
+ typedef Ordering::CompValueThenAddressGreater<MapType, MapType> MaxOrderArb; // arbitrary on ties
+ typedef Ordering::CompValueThenAddressLesser<MapType, MapType> MinOrderArb; // arbitrary on ties
+ typedef Ordering::CompValueThenAddressGreater<MapType, MapType> MaxOrderRand; // on ties, return a random element (keep all elements around)
+ typedef Ordering::CompValueThenAddressLesser<MapType, MapType> MinOrderRand; // on ties, return a random element (keep all elements around)
+ typedef Bed::ScoreThenGenomicCompareGreater<MapType, MapType> MaxOrderStable; // on ties, return first genomic element observed (only first kept on value+genomic ties)
+ typedef Bed::ScoreThenGenomicCompareLesser<MapType, MapType> MinOrderStable; // on ties, return first genomic element observed (only first kept on value+genomic ties)
typedef Visitors::Average<ProcessScorePrecision, BaseClass> Average;
typedef Visitors::CoeffVariation<ProcessScorePrecision, BaseClass> CoeffVariation;
typedef Visitors::Count<ProcessScore, BaseClass> Count;
typedef Visitors::RollingKthAverage<ProcessScorePrecision, BaseClass, Ext::ArgumentError> KthAverage;
- typedef Visitors::Extreme<ProcessScorePrecision, BaseClass, MaxOrder> Max;
+ typedef Visitors::Extreme<ProcessScorePrecision, BaseClass, MaxOrderArb> Max;
typedef Visitors::Indicator<ProcessScore, BaseClass> Indicator;
typedef Visitors::Median<ProcessScorePrecision, BaseClass> Median;
typedef Visitors::MedianAbsoluteDeviation<ProcessScorePrecision, BaseClass> MedianAbsoluteDeviation;
- typedef Visitors::Extreme<ProcessScorePrecision, BaseClass, MinOrder> Min;
+ typedef Visitors::Extreme<ProcessScorePrecision, BaseClass, MinOrderArb> Min;
typedef Visitors::StdDev<ProcessScorePrecision, BaseClass> StdDev;
typedef Visitors::Sum<ProcessScorePrecision, BaseClass> Sum;
typedef Visitors::TrimmedMean<ProcessScorePrecision, BaseClass, Ext::ArgumentError> TMean;
+ typedef Visitors::WeightedAverage<ProcessScorePrecision, BaseClass> WMean;
typedef Visitors::Variance<ProcessScorePrecision, BaseClass> Variance;
- typedef Visitors::Extreme<ProcessOne, BaseClass, MaxOrder> MaxElement;
- typedef Visitors::Extreme<ProcessOne, BaseClass, MinOrder> MinElement;
+ typedef Visitors::Extreme<ProcessOne, BaseClass, MaxOrderRand, Visitors::RandTie> MaxElementRand;
+ typedef Visitors::Extreme<ProcessOne, BaseClass, MinOrderRand, Visitors::RandTie> MinElementRand;
+ typedef Visitors::Extreme<ProcessOne, BaseClass, MaxOrderStable> MaxElementStable;
+ typedef Visitors::Extreme<ProcessOne, BaseClass, MinOrderStable> MinElementStable;
typedef Visitors::BedSpecific::EchoMapBed<ProcessRangeDelimAll, BaseClass> EchoMapAll;
typedef Visitors::BedSpecific::EchoMapBed<ProcessRangeDelimID, BaseClass> EchoMapID;
diff --git a/applications/bed/bedops/src/BedPadReader.hpp b/applications/bed/bedops/src/BedPadReader.hpp
index 23b40d5..7995181 100644
--- a/applications/bed/bedops/src/BedPadReader.hpp
+++ b/applications/bed/bedops/src/BedPadReader.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/applications/bed/bedops/src/Bedops.cpp b/applications/bed/bedops/src/Bedops.cpp
index ae38306..9fb741f 100644
--- a/applications/bed/bedops/src/Bedops.cpp
+++ b/applications/bed/bedops/src/Bedops.cpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -33,6 +33,7 @@
#include <numeric>
#include <queue>
#include <string>
+#include <type_traits>
#include <utility>
#include <vector>
@@ -749,7 +750,12 @@ void doSymmetricDifference(BedFiles& bedFiles) {
// doUnionAll()
//==============
template <typename BedFiles>
-typename GetType<BedFiles>::BedType* nextUnionAllLine(BedFiles&);
+typename std::enable_if<GetType<BedFiles>::BedType::UseRest, typename GetType<BedFiles>::BedType>::type*
+nextUnionAllLine(BedFiles&); /* requires BedType::UseRest to be true */
+
+template <typename BedFiles>
+typename std::enable_if<!GetType<BedFiles>::BedType::UseRest, typename GetType<BedFiles>::BedType>::type*
+nextUnionAllLine(BedFiles&); /* unimplemented on purpose */
template <typename BedFiles>
void doUnionAll(BedFiles& bedFiles) {
@@ -1464,7 +1470,8 @@ std::pair<bool, typename GetType<BedFiles>::BedType*>
// nextUnionAllLine()
//====================
template <typename BedFiles>
-typename GetType<BedFiles>::BedType* nextUnionAllLine(BedFiles& bedFiles) {
+typename std::enable_if<GetType<BedFiles>::BedType::UseRest, typename GetType<BedFiles>::BedType>::type*
+nextUnionAllLine(BedFiles& bedFiles) {
// Find next minimum entry between all files
typedef typename GetType<BedFiles>::BedType BedType;
static BedType* const zero = static_cast<BedType*>(0);
@@ -1490,9 +1497,16 @@ typename GetType<BedFiles>::BedType* nextUnionAllLine(BedFiles& bedFiles) {
if ( next->start() < first->start() ) {
first = next;
marker = i;
- } else if ( next->start() == first->start() && next->end() < first->end() ) {
- first = next;
- marker = i;
+ } else if ( next->start() == first->start() ) {
+ if ( next->end() < first->end() ) {
+ first = next;
+ marker = i;
+ } else if ( next->end() == first->end() ) {
+ if ( strcmp(next->rest(), first->rest()) < 0 ) {
+ first = next;
+ marker = i;
+ }
+ }
}
}
}
@@ -1514,6 +1528,7 @@ void selectWork(const Input& input, BedFiles& bedFiles) {
switch ( modeType ) {
case CHOP:
doChop(bedFiles, input.ChopChunkSize(), input.ChopStaggerSize(), input.ChopExcludeShort());
+ break;
case COMPLEMENT:
doComplement(bedFiles, input.ComplementFullLeft());
break;
@@ -1533,7 +1548,8 @@ void selectWork(const Input& input, BedFiles& bedFiles) {
doSymmetricDifference(bedFiles);
break;
case UNIONALL:
- doUnionAll(bedFiles);
+ if ( GetType<BedFiles>::BedType::UseRest )
+ doUnionAll(bedFiles);
break;
default:
throw(Ext::ProgramError("Unsupported mode"));
diff --git a/applications/bed/bedops/src/Input.hpp b/applications/bed/bedops/src/Input.hpp
index e70518a..1a6e710 100644
--- a/applications/bed/bedops/src/Input.hpp
+++ b/applications/bed/bedops/src/Input.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/applications/bed/bedops/src/Makefile.darwin b/applications/bed/bedops/src/Makefile.darwin
index 2561311..4a564f7 100644
--- a/applications/bed/bedops/src/Makefile.darwin
+++ b/applications/bed/bedops/src/Makefile.darwin
@@ -31,7 +31,7 @@ BLDFLAGS = -O3 ${STDFLAGS}
FLAGS = $(BLDFLAGS) $(OBJDIR)/NaN.o $(OBJDIR)/starchConstants.o $(OBJDIR)/starchFileHelpers.o $(OBJDIR)/starchHelpers.o $(OBJDIR)/starchMetadataHelpers.o $(OBJDIR)/unstarchHelpers.o $(OBJDIR)/starchSha1Digest.o $(OBJDIR)/starchBase64Coding.o ${LIBLOCATION} ${INCLUDES}
-DFLAGS = -g -O0 -DDEBUG_VERBOSE=1 ${STDFLAGS} $(OBJDIR)/NaN.o $(OBJDIR)/starchConstants.o $(OBJDIR)/starchFileHelpers.o $(OBJDIR)/starchHelpers.o $(OBJDIR)/starchMetadataHelpers.o $(OBJDIR)/unstarchHelpers.o $(OBJDIR)/starchSha1Digest.o $(OBJDIR)/starchBase64Coding.o ${LIBLOCATION} ${INCLUDES}
+DFLAGS = -g -O0 -DDEBUG_VERBOSE=1 -DDEBUG=1 ${STDFLAGS} $(OBJDIR)/NaN.o $(OBJDIR)/starchConstants.o $(OBJDIR)/starchFileHelpers.o $(OBJDIR)/starchHelpers.o $(OBJDIR)/starchMetadataHelpers.o $(OBJDIR)/unstarchHelpers.o $(OBJDIR)/starchSha1Digest.o $(OBJDIR)/starchBase64Coding.o ${LIBLOCATION} ${INCLUDES}
GPROFFLAGS = -O -pg ${STDFLAGS} $(OBJDIR)/NaN.o $(OBJDIR)/starchConstants.o $(OBJDIR)/starchFileHelpers.o $(OBJDIR)/starchHelpers.o $(OBJDIR)/starchMetadataHelpers.o $(OBJDIR)/unstarchHelpers.o $(OBJDIR)/starchSha1Digest.o $(OBJDIR)/starchBase64Coding.o ${LIBLOCATION} ${INCLUDES}
diff --git a/applications/bed/bedops/test/Regression.java b/applications/bed/bedops/test/Regression.java
index 3d929c1..b7bdb2f 100644
--- a/applications/bed/bedops/test/Regression.java
+++ b/applications/bed/bedops/test/Regression.java
@@ -1,5 +1,5 @@
// Author
-// Shane Neph : University of Washington
+// Shane Neph : 2006
import org.w3c.dom.*;
import javax.xml.parsers.*;
diff --git a/applications/bed/bedops/test/TestPlan.xml b/applications/bed/bedops/test/TestPlan.xml
index dec2402..e474029 100644
--- a/applications/bed/bedops/test/TestPlan.xml
+++ b/applications/bed/bedops/test/TestPlan.xml
@@ -1910,4 +1910,59 @@
<CALL>--chop 100 -x --stagger 53 chop1.test chop2.test</CALL>
<OUTPUT name="chop4xs.results"/>
</TEST>
+
+ <!-- *******CHOP TEST******* -->
+ <!-- Creates chop3.test -->
+ <!-- Creates chop5.results -->
+ <!-- testing stagger and -x options of -w/chop -->
+ <TEST order="62">
+ <INPUT name="chop3.test">
+ chr1 10 15 id-1
+ chr2 10 15
+ chr3 10 15
+ chr9 10 15
+ chrM 10 15
+ chrX 10 15
+ chrY 10 15 id-2
+ </INPUT>
+ <ANSWER>
+ chr1 10 11
+ chr1 11 12
+ chr1 12 13
+ chr1 13 14
+ chr1 14 15
+ chr2 10 11
+ chr2 11 12
+ chr2 12 13
+ chr2 13 14
+ chr2 14 15
+ chr3 10 11
+ chr3 11 12
+ chr3 12 13
+ chr3 13 14
+ chr3 14 15
+ chr9 10 11
+ chr9 11 12
+ chr9 12 13
+ chr9 13 14
+ chr9 14 15
+ chrM 10 11
+ chrM 11 12
+ chrM 12 13
+ chrM 13 14
+ chrM 14 15
+ chrX 10 11
+ chrX 11 12
+ chrX 12 13
+ chrX 13 14
+ chrX 14 15
+ chrY 10 11
+ chrY 11 12
+ chrY 12 13
+ chrY 13 14
+ chrY 14 15
+ </ANSWER>
+ <CALL>--chop</CALL>
+ <OUTPUT name="chop5.results"/>
+ </TEST>
</TEST_PLAN>
diff --git a/applications/bed/closestfeats/doc/Usage.Statement.Version1.2 b/applications/bed/closestfeats/doc/Usage.Statement.Version1.2
deleted file mode 100644
index f94cda5..0000000
--- a/applications/bed/closestfeats/doc/Usage.Statement.Version1.2
+++ /dev/null
@@ -1,45 +0,0 @@
-closest-features
- version: 1.2
- authors: Shane Neph & Scott Kuehn
-
-Usage: closestFeatures [Process-Flags] [Output-Option] <input-file> <query-file>
- All input files must be sorted per sort-bed.
- May use '-' for a file to indicate reading from standard input.
-
- For every element in <input-file>, determine the two elements from <query-file> falling
- nearest to its left and right edges. An 'Output-Option' may be set to report results
- in any one of a variety of ways.
-
- Process Flags:
- --ec : Error check all input files (slower).
- --delim <delim> : Change output delimiter from '|' to <delim> between columns (e.g. '\t')
- --no-overlaps : Overlapping elements from <query-file> will not be reported.
- Useful, for example, with --5P if reported elements must be upstream.
- --no-ref : Do not echo elements from <input-file>.
- --dist : Print the signed distance(s) to the <input-file> element as an additional
- column(s) of output. An overlapping element has a distance of 0.
-
- Output-Option (At most, one may be selected.):
- Without --no-ref, all output options include the <input-file> element as the first field.
- By default, also print the two elements from <query-file> that lie genomically closest
- to the left and right edges of the <input-file> element, respectively.
-
- --5P : Print closest element to 5' edge of <input-file> element.
- --3P : Print closest element to 3' edge of <input-file> element.
- --shortest : Print closest element only. Ties go the left element.
- --left : Print closest element to left edge of <input-file> element only.
- --right : Print closest element to right edge of <input-file> element only.
-
- --5P and --3P require at least 6 fields in the <input-file>, where the 6th column
- indicates strand (either + or -), while the <query-file> requires only 3 columns.
- All other options, including the default behavior, require 3 columns in both files.
-
- NOTES:
- If an element from <query-file> overlaps the <input-file> element, its distance is zero.
- An overlapping element take precedence over all non-overlapping elements. This is true
- even when the overlapping element's edge-to-edge distance to the <input-file>'s element
- is greater than the edge-to-edge distance from a non-overlapping element.
- Overlapping elements may be ignored completely (no precedence) with --no-overlaps.
- Elements reported as genomically closest to the left and right edges are never the same.
- When no qualifying element from <query-file> exists as a closest feature, 'NA' is reported.
-
diff --git a/applications/bed/closestfeats/src/BedReader.hpp b/applications/bed/closestfeats/src/BedReader.hpp
index a1ee449..25ce6bb 100644
--- a/applications/bed/closestfeats/src/BedReader.hpp
+++ b/applications/bed/closestfeats/src/BedReader.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/applications/bed/closestfeats/src/ClosestFeature.cpp b/applications/bed/closestfeats/src/ClosestFeature.cpp
index 9b042f5..7e08a00 100644
--- a/applications/bed/closestfeats/src/ClosestFeature.cpp
+++ b/applications/bed/closestfeats/src/ClosestFeature.cpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/applications/bed/closestfeats/src/Input.hpp b/applications/bed/closestfeats/src/Input.hpp
index c3581d0..8d53610 100644
--- a/applications/bed/closestfeats/src/Input.hpp
+++ b/applications/bed/closestfeats/src/Input.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/applications/bed/closestfeats/src/Printers.hpp b/applications/bed/closestfeats/src/Printers.hpp
index 0153e55..798dc13 100644
--- a/applications/bed/closestfeats/src/Printers.hpp
+++ b/applications/bed/closestfeats/src/Printers.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/applications/bed/conversion/src/convert2bed.c b/applications/bed/conversion/src/convert2bed.c
index 8f11b51..b3d7bfe 100644
--- a/applications/bed/conversion/src/convert2bed.c
+++ b/applications/bed/conversion/src/convert2bed.c
@@ -1,9 +1,9 @@
/*
convert2bed.c
-----------------------------------------------------------------------
- Copyright (C) 2014-2016 Alex Reynolds
+ Copyright (C) 2014-2017 Alex Reynolds
- wig2bed components, (C) 2011-2016 Scott Kuehn and Shane Neph
+ wig2bed components, (C) 2011-2017 Scott Kuehn and Shane Neph
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -187,6 +187,7 @@ c2b_init_generic_conversion(c2b_pipeset_t *p, void(*to_bed_line_functor)(char *,
char bed_unsorted2bed_sorted_cmd[C2B_MAX_LINE_LENGTH_VALUE];
char bed_sorted2starch_cmd[C2B_MAX_LINE_LENGTH_VALUE];
void (*generic2bed_unsorted_line_functor)(char *, ssize_t *, char *, ssize_t) = to_bed_line_functor;
+ ssize_t buffer_size = C2B_THREAD_IO_BUFFER_SIZE;
int errsv = 0;
if ((!c2b_globals.sort->is_enabled) && (c2b_globals.output_format_idx == BED_FORMAT)) {
@@ -197,6 +198,7 @@ c2b_init_generic_conversion(c2b_pipeset_t *p, void(*to_bed_line_functor)(char *,
cat2generic_stage.description = "Generic data from stdin";
cat2generic_stage.pid = 0;
cat2generic_stage.status = 0;
+ cat2generic_stage.buffer_size = buffer_size;
generic2bed_unsorted_stage.pipeset = p;
generic2bed_unsorted_stage.line_functor = generic2bed_unsorted_line_functor;
@@ -205,6 +207,7 @@ c2b_init_generic_conversion(c2b_pipeset_t *p, void(*to_bed_line_functor)(char *,
generic2bed_unsorted_stage.description = "Generic data to unsorted BED";
generic2bed_unsorted_stage.pid = 0;
generic2bed_unsorted_stage.status = 0;
+ generic2bed_unsorted_stage.buffer_size = buffer_size;
bed_unsorted2stdout_stage.pipeset = p;
bed_unsorted2stdout_stage.line_functor = NULL;
@@ -213,6 +216,7 @@ c2b_init_generic_conversion(c2b_pipeset_t *p, void(*to_bed_line_functor)(char *,
bed_unsorted2stdout_stage.description = "Unsorted BED to stdout";
bed_unsorted2stdout_stage.pid = 0;
bed_unsorted2stdout_stage.status = 0;
+ bed_unsorted2stdout_stage.buffer_size = buffer_size;
}
else if (c2b_globals.output_format_idx == BED_FORMAT) {
cat2generic_stage.pipeset = p;
@@ -222,6 +226,7 @@ c2b_init_generic_conversion(c2b_pipeset_t *p, void(*to_bed_line_functor)(char *,
cat2generic_stage.description = "Generic data from stdin";
cat2generic_stage.pid = 0;
cat2generic_stage.status = 0;
+ cat2generic_stage.buffer_size = buffer_size;
generic2bed_unsorted_stage.pipeset = p;
generic2bed_unsorted_stage.line_functor = generic2bed_unsorted_line_functor;
@@ -230,6 +235,7 @@ c2b_init_generic_conversion(c2b_pipeset_t *p, void(*to_bed_line_functor)(char *,
generic2bed_unsorted_stage.description = "Generic data to unsorted BED";
generic2bed_unsorted_stage.pid = 0;
generic2bed_unsorted_stage.status = 0;
+ generic2bed_unsorted_stage.buffer_size = buffer_size;
bed_unsorted2bed_sorted_stage.pipeset = p;
bed_unsorted2bed_sorted_stage.line_functor = NULL;
@@ -238,6 +244,7 @@ c2b_init_generic_conversion(c2b_pipeset_t *p, void(*to_bed_line_functor)(char *,
bed_unsorted2bed_sorted_stage.description = "Unsorted BED to sorted BED";
bed_unsorted2bed_sorted_stage.pid = 0;
bed_unsorted2bed_sorted_stage.status = 0;
+ bed_unsorted2bed_sorted_stage.buffer_size = buffer_size;
bed_sorted2stdout_stage.pipeset = p;
bed_sorted2stdout_stage.line_functor = NULL;
@@ -246,6 +253,7 @@ c2b_init_generic_conversion(c2b_pipeset_t *p, void(*to_bed_line_functor)(char *,
bed_sorted2stdout_stage.description = "Sorted BED to stdout";
bed_sorted2stdout_stage.pid = 0;
bed_sorted2stdout_stage.status = 0;
+ bed_sorted2stdout_stage.buffer_size = buffer_size;
}
else if (c2b_globals.output_format_idx == STARCH_FORMAT) {
cat2generic_stage.pipeset = p;
@@ -255,6 +263,7 @@ c2b_init_generic_conversion(c2b_pipeset_t *p, void(*to_bed_line_functor)(char *,
cat2generic_stage.description = "Generic data from stdin";
cat2generic_stage.pid = 0;
cat2generic_stage.status = 0;
+ cat2generic_stage.buffer_size = buffer_size;
generic2bed_unsorted_stage.pipeset = p;
generic2bed_unsorted_stage.line_functor = generic2bed_unsorted_line_functor;
@@ -263,6 +272,7 @@ c2b_init_generic_conversion(c2b_pipeset_t *p, void(*to_bed_line_functor)(char *,
generic2bed_unsorted_stage.description = "Generic data to unsorted BED";
generic2bed_unsorted_stage.pid = 0;
generic2bed_unsorted_stage.status = 0;
+ generic2bed_unsorted_stage.buffer_size = buffer_size;
bed_unsorted2bed_sorted_stage.pipeset = p;
bed_unsorted2bed_sorted_stage.line_functor = NULL;
@@ -271,6 +281,7 @@ c2b_init_generic_conversion(c2b_pipeset_t *p, void(*to_bed_line_functor)(char *,
bed_unsorted2bed_sorted_stage.description = "Unsorted BED to sorted BED";
bed_unsorted2bed_sorted_stage.pid = 0;
bed_unsorted2bed_sorted_stage.status = 0;
+ bed_unsorted2bed_sorted_stage.buffer_size = buffer_size;
bed_sorted2starch_stage.pipeset = p;
bed_sorted2starch_stage.line_functor = NULL;
@@ -279,6 +290,7 @@ c2b_init_generic_conversion(c2b_pipeset_t *p, void(*to_bed_line_functor)(char *,
bed_sorted2stdout_stage.description = "Sorted BED to Starch";
bed_sorted2starch_stage.pid = 0;
bed_sorted2starch_stage.status = 0;
+ bed_sorted2starch_stage.buffer_size = buffer_size;
starch2stdout_stage.pipeset = p;
starch2stdout_stage.line_functor = NULL;
@@ -287,6 +299,7 @@ c2b_init_generic_conversion(c2b_pipeset_t *p, void(*to_bed_line_functor)(char *,
starch2stdout_stage.description = "Starch to stdout";
starch2stdout_stage.pid = 0;
starch2stdout_stage.status = 0;
+ starch2stdout_stage.buffer_size = buffer_size;
}
else {
fprintf(stderr, "Error: Unknown conversion parameter combination\n");
@@ -472,6 +485,7 @@ c2b_init_bam_conversion(c2b_pipeset_t *p)
char bed_unsorted2bed_sorted_cmd[C2B_MAX_LINE_LENGTH_VALUE];
char bed_sorted2starch_cmd[C2B_MAX_LINE_LENGTH_VALUE];
void (*sam2bed_unsorted_line_functor)(char *, ssize_t *, char *, ssize_t) = NULL;
+ ssize_t buffer_size = C2B_THREAD_IO_BUFFER_SIZE;
int errsv = errno;
sam2bed_unsorted_line_functor = (!c2b_globals.split_flag ?
@@ -486,6 +500,7 @@ c2b_init_bam_conversion(c2b_pipeset_t *p)
bam2sam_stage.description = "BAM data from stdin to SAM";
bam2sam_stage.pid = 0;
bam2sam_stage.status = 0;
+ bam2sam_stage.buffer_size = buffer_size;
sam2bed_unsorted_stage.pipeset = p;
sam2bed_unsorted_stage.line_functor = sam2bed_unsorted_line_functor;
@@ -494,6 +509,7 @@ c2b_init_bam_conversion(c2b_pipeset_t *p)
sam2bed_unsorted_stage.description = "SAM to unsorted BED";
sam2bed_unsorted_stage.pid = 0;
sam2bed_unsorted_stage.status = 0;
+ sam2bed_unsorted_stage.buffer_size = buffer_size;
bed_unsorted2stdout_stage.pipeset = p;
bed_unsorted2stdout_stage.line_functor = NULL;
@@ -502,6 +518,7 @@ c2b_init_bam_conversion(c2b_pipeset_t *p)
bed_unsorted2stdout_stage.description = "Unsorted BED to stdout";
bed_unsorted2stdout_stage.pid = 0;
bed_unsorted2stdout_stage.status = 0;
+ bed_unsorted2stdout_stage.buffer_size = buffer_size;
}
else if (c2b_globals.output_format_idx == BED_FORMAT) {
bam2sam_stage.pipeset = p;
@@ -511,6 +528,7 @@ c2b_init_bam_conversion(c2b_pipeset_t *p)
bam2sam_stage.description = "BAM data from stdin to SAM";
bam2sam_stage.pid = 0;
bam2sam_stage.status = 0;
+ bam2sam_stage.buffer_size = buffer_size;
sam2bed_unsorted_stage.pipeset = p;
sam2bed_unsorted_stage.line_functor = sam2bed_unsorted_line_functor;
@@ -519,6 +537,7 @@ c2b_init_bam_conversion(c2b_pipeset_t *p)
sam2bed_unsorted_stage.description = "SAM to unsorted BED";
sam2bed_unsorted_stage.pid = 0;
sam2bed_unsorted_stage.status = 0;
+ sam2bed_unsorted_stage.buffer_size = buffer_size;
bed_unsorted2bed_sorted_stage.pipeset = p;
bed_unsorted2bed_sorted_stage.line_functor = NULL;
@@ -527,6 +546,7 @@ c2b_init_bam_conversion(c2b_pipeset_t *p)
bed_unsorted2bed_sorted_stage.description = "Unsorted BED to sorted BED";
bed_unsorted2bed_sorted_stage.pid = 0;
bed_unsorted2bed_sorted_stage.status = 0;
+ bed_unsorted2bed_sorted_stage.buffer_size = buffer_size;
bed_sorted2stdout_stage.pipeset = p;
bed_sorted2stdout_stage.line_functor = NULL;
@@ -535,6 +555,7 @@ c2b_init_bam_conversion(c2b_pipeset_t *p)
bed_sorted2stdout_stage.description = "Sorted BED to stdout";
bed_sorted2stdout_stage.pid = 0;
bed_sorted2stdout_stage.status = 0;
+ bed_sorted2stdout_stage.buffer_size = buffer_size;
}
else if (c2b_globals.output_format_idx == STARCH_FORMAT) {
bam2sam_stage.pipeset = p;
@@ -544,7 +565,8 @@ c2b_init_bam_conversion(c2b_pipeset_t *p)
bam2sam_stage.description = "BAM data from stdin to SAM";
bam2sam_stage.pid = 0;
bam2sam_stage.status = 0;
-
+ bam2sam_stage.buffer_size = buffer_size;
+
sam2bed_unsorted_stage.pipeset = p;
sam2bed_unsorted_stage.line_functor = sam2bed_unsorted_line_functor;
sam2bed_unsorted_stage.src = 0;
@@ -552,6 +574,7 @@ c2b_init_bam_conversion(c2b_pipeset_t *p)
sam2bed_unsorted_stage.description = "SAM to unsorted BED";
sam2bed_unsorted_stage.pid = 0;
sam2bed_unsorted_stage.status = 0;
+ sam2bed_unsorted_stage.buffer_size = buffer_size;
bed_unsorted2bed_sorted_stage.pipeset = p;
bed_unsorted2bed_sorted_stage.line_functor = NULL;
@@ -560,6 +583,7 @@ c2b_init_bam_conversion(c2b_pipeset_t *p)
bed_unsorted2bed_sorted_stage.description = "Unsorted BED to sorted BED";
bed_unsorted2bed_sorted_stage.pid = 0;
bed_unsorted2bed_sorted_stage.status = 0;
+ bed_unsorted2bed_sorted_stage.buffer_size = buffer_size;
bed_sorted2starch_stage.pipeset = p;
bed_sorted2starch_stage.line_functor = NULL;
@@ -568,6 +592,7 @@ c2b_init_bam_conversion(c2b_pipeset_t *p)
bed_sorted2starch_stage.description = "Sorted BED to Starch";
bed_sorted2starch_stage.pid = 0;
bed_sorted2starch_stage.status = 0;
+ bed_sorted2starch_stage.buffer_size = buffer_size;
starch2stdout_stage.pipeset = p;
starch2stdout_stage.line_functor = NULL;
@@ -576,6 +601,7 @@ c2b_init_bam_conversion(c2b_pipeset_t *p)
starch2stdout_stage.description = "Starch to stdout";
starch2stdout_stage.pid = 0;
starch2stdout_stage.status = 0;
+ starch2stdout_stage.buffer_size = buffer_size;
}
else {
fprintf(stderr, "Error: Unknown BAM conversion parameter combination\n");
@@ -834,9 +860,9 @@ c2b_cmd_starch_bed(char *cmd)
#ifdef DEBUG
fprintf(stderr, "Debug: c2b_globals.starch->bzip2: [%d]\n", c2b_globals.starch->bzip2);
- fprintf(stderr, "Debug: c2b_globals.starch->gzip: [%d]\n", c2b_globals.starch->gzip);
- fprintf(stderr, "Debug: c2b_globals.starch->note: [%s]\n", c2b_globals.starch->note);
- fprintf(stderr, "Debug: starch_args: [%s]\n", starch_args);
+ fprintf(stderr, "Debug: c2b_globals.starch->gzip: [%d]\n", c2b_globals.starch->gzip);
+ fprintf(stderr, "Debug: c2b_globals.starch->note: [%s]\n", c2b_globals.starch->note);
+ fprintf(stderr, "Debug: starch_args: [%s]\n", starch_args);
#endif
if (c2b_globals.starch->note) {
@@ -870,6 +896,107 @@ c2b_cmd_starch_bed(char *cmd)
}
static void
+c2b_gtf_init_element(c2b_gtf_t **e)
+{
+ *e = malloc(sizeof(c2b_gtf_t));
+ if (!*e) {
+ fprintf(stderr, "Error: Could not allocate space for GTF element pointer\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+
+ (*e)->seqname = NULL, (*e)->seqname = malloc(C2B_GTF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->seqname)));
+ if (!(*e)->seqname) {
+ fprintf(stderr, "Error: Could not allocate space for GTF element seqname malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->seqname_capacity = C2B_GTF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->source = NULL, (*e)->source = malloc(C2B_GTF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->source)));
+ if (!(*e)->source) {
+ fprintf(stderr, "Error: Could not allocate space for GTF element source malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->source_capacity = C2B_GTF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->feature = NULL, (*e)->feature = malloc(C2B_GTF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->feature)));
+ if (!(*e)->feature) {
+ fprintf(stderr, "Error: Could not allocate space for GTF element feature malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->feature_capacity = C2B_GTF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->start = 0;
+ (*e)->end = 0;
+
+ (*e)->score = NULL, (*e)->score = malloc(C2B_GTF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->score)));
+ if (!(*e)->score) {
+ fprintf(stderr, "Error: Could not allocate space for GTF element score malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->score_capacity = C2B_GTF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->strand = NULL, (*e)->strand = malloc(C2B_GTF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->strand)));
+ if (!(*e)->strand) {
+ fprintf(stderr, "Error: Could not allocate space for GTF element strand malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->strand_capacity = C2B_GTF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->frame = NULL, (*e)->frame = malloc(C2B_GTF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->frame)));
+ if (!(*e)->frame) {
+ fprintf(stderr, "Error: Could not allocate space for GTF element frame malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->frame_capacity = C2B_GTF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->attributes = NULL, (*e)->attributes = malloc(C2B_GTF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->attributes)));
+ if (!(*e)->attributes) {
+ fprintf(stderr, "Error: Could not allocate space for GTF element attributes malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->attributes_capacity = C2B_GTF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->id = NULL, (*e)->id = malloc(C2B_GTF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->id)));
+ if (!(*e)->id) {
+ fprintf(stderr, "Error: Could not allocate space for GTF element id malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->id_capacity = C2B_GTF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->comments = NULL, (*e)->comments = malloc(C2B_GTF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->comments)));
+ if (!(*e)->comments) {
+ fprintf(stderr, "Error: Could not allocate space for GTF element comments malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->comments_capacity = C2B_GTF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+}
+
+static void
+c2b_gtf_delete_element(c2b_gtf_t *e)
+{
+ if (e->seqname) { free(e->seqname), e->seqname = NULL; }
+ if (e->source) { free(e->source), e->source = NULL; }
+ if (e->feature) { free(e->feature), e->feature = NULL; }
+ if (e->score) { free(e->score), e->score = NULL; }
+ if (e->strand) { free(e->strand), e->strand = NULL; }
+ if (e->frame) { free(e->frame), e->frame = NULL; }
+ if (e->attributes) { free(e->attributes), e->attributes = NULL; }
+ if (e->id) { free(e->id), e->id = NULL; }
+ if (e->comments) { free(e->comments), e->comments = NULL; }
+ if (e) { free(e), e = NULL; }
+}
+
+static void
c2b_line_convert_gtf_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src, ssize_t src_size)
{
ssize_t gtf_field_offsets[C2B_MAX_FIELD_COUNT_VALUE];
@@ -924,95 +1051,171 @@ c2b_line_convert_gtf_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src,
}
/* 0 - seqname */
- char seqname_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t seqname_size = gtf_field_offsets[0];
- memcpy(seqname_str, src, seqname_size);
- seqname_str[seqname_size] = '\0';
+ if (seqname_size >= c2b_globals.gtf->element->seqname_capacity) {
+ char *seqname_resized = NULL;
+ seqname_resized = realloc(c2b_globals.gtf->element->seqname, seqname_size + 1);
+ if (seqname_resized) {
+ c2b_globals.gtf->element->seqname = seqname_resized;
+ c2b_globals.gtf->element->seqname_capacity = seqname_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize SEQNAME string in GTF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.gtf->element->seqname, src, seqname_size);
+ c2b_globals.gtf->element->seqname[seqname_size] = '\0';
/* 1 - source */
- char source_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t source_size = gtf_field_offsets[1] - gtf_field_offsets[0] - 1;
- memcpy(source_str, src + gtf_field_offsets[0] + 1, source_size);
- source_str[source_size] = '\0';
+ if (source_size >= c2b_globals.gtf->element->source_capacity) {
+ char *source_resized = NULL;
+ source_resized = realloc(c2b_globals.gtf->element->source, source_size + 1);
+ if (source_resized) {
+ c2b_globals.gtf->element->source = source_resized;
+ c2b_globals.gtf->element->source_capacity = source_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize SOURCE string in GTF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.gtf->element->source, src + gtf_field_offsets[0] + 1, source_size);
+ c2b_globals.gtf->element->source[source_size] = '\0';
/* 2 - feature */
- char feature_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t feature_size = gtf_field_offsets[2] - gtf_field_offsets[1] - 1;
- memcpy(feature_str, src + gtf_field_offsets[1] + 1, feature_size);
- feature_str[feature_size] = '\0';
+ if (feature_size >= c2b_globals.gtf->element->feature_capacity) {
+ char *feature_resized = NULL;
+ feature_resized = realloc(c2b_globals.gtf->element->feature, feature_size + 1);
+ if (feature_resized) {
+ c2b_globals.gtf->element->feature = feature_resized;
+ c2b_globals.gtf->element->feature_capacity = feature_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize FEATURE string in GTF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.gtf->element->feature, src + gtf_field_offsets[1] + 1, feature_size);
+ c2b_globals.gtf->element->feature[feature_size] = '\0';
/* 3 - start */
char start_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t start_size = gtf_field_offsets[3] - gtf_field_offsets[2] - 1;
memcpy(start_str, src + gtf_field_offsets[2] + 1, start_size);
start_str[start_size] = '\0';
- uint64_t start_val = strtoull(start_str, NULL, 10);
+ c2b_globals.gtf->element->start = strtoull(start_str, NULL, 10);
/* 4 - end */
char end_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t end_size = gtf_field_offsets[4] - gtf_field_offsets[3] - 1;
memcpy(end_str, src + gtf_field_offsets[3] + 1, end_size);
end_str[end_size] = '\0';
- uint64_t end_val = strtoull(end_str, NULL, 10);
+ c2b_globals.gtf->element->end = strtoull(end_str, NULL, 10);
/* 5 - score */
- char score_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t score_size = gtf_field_offsets[5] - gtf_field_offsets[4] - 1;
- memcpy(score_str, src + gtf_field_offsets[4] + 1, score_size);
- score_str[score_size] = '\0';
+ if (score_size >= c2b_globals.gtf->element->score_capacity) {
+ char *score_resized = NULL;
+ score_resized = realloc(c2b_globals.gtf->element->score, score_size + 1);
+ if (score_resized) {
+ c2b_globals.gtf->element->score = score_resized;
+ c2b_globals.gtf->element->score_capacity = score_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize SCORE string in GTF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.gtf->element->score, src + gtf_field_offsets[4] + 1, score_size);
+ c2b_globals.gtf->element->score[score_size] = '\0';
/* 6 - strand */
- char strand_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t strand_size = gtf_field_offsets[6] - gtf_field_offsets[5] - 1;
- memcpy(strand_str, src + gtf_field_offsets[5] + 1, strand_size);
- strand_str[strand_size] = '\0';
+ if (strand_size >= c2b_globals.gtf->element->strand_capacity) {
+ char *strand_resized = NULL;
+ strand_resized = realloc(c2b_globals.gtf->element->strand, strand_size + 1);
+ if (strand_resized) {
+ c2b_globals.gtf->element->strand = strand_resized;
+ c2b_globals.gtf->element->strand_capacity = strand_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize STRAND string in GTF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.gtf->element->strand, src + gtf_field_offsets[5] + 1, strand_size);
+ c2b_globals.gtf->element->strand[strand_size] = '\0';
/* 7 - frame */
- char frame_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t frame_size = gtf_field_offsets[7] - gtf_field_offsets[6] - 1;
- memcpy(frame_str, src + gtf_field_offsets[6] + 1, frame_size);
- frame_str[frame_size] = '\0';
+ if (frame_size >= c2b_globals.gtf->element->frame_capacity) {
+ char *frame_resized = NULL;
+ frame_resized = realloc(c2b_globals.gtf->element->frame, frame_size + 1);
+ if (frame_resized) {
+ c2b_globals.gtf->element->frame = frame_resized;
+ c2b_globals.gtf->element->frame_capacity = frame_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize FRAME string in GTF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.gtf->element->frame, src + gtf_field_offsets[6] + 1, frame_size);
+ c2b_globals.gtf->element->frame[frame_size] = '\0';
/* 8 - attributes */
- char attributes_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t attributes_size = gtf_field_offsets[8] - gtf_field_offsets[7] - 1;
- memcpy(attributes_str, src + gtf_field_offsets[7] + 1, attributes_size);
- attributes_str[attributes_size] = '\0';
+ if (attributes_size >= c2b_globals.gtf->element->attributes_capacity) {
+ char *attributes_resized = NULL;
+ attributes_resized = realloc(c2b_globals.gtf->element->attributes, attributes_size * 2);
+ if (attributes_resized) {
+ c2b_globals.gtf->element->attributes = attributes_resized;
+ c2b_globals.gtf->element->attributes_capacity = attributes_size * 2;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize ATTRIBUTES string in GTF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.gtf->element->attributes, src + gtf_field_offsets[7] + 1, attributes_size);
+ c2b_globals.gtf->element->attributes[attributes_size] = '\0';
/* 9 - comments */
- char comments_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t comments_size = 0;
if (gtf_field_idx == 9) {
comments_size = gtf_field_offsets[9] - gtf_field_offsets[8] - 1;
- memcpy(comments_str, src + gtf_field_offsets[8] + 1, comments_size);
- comments_str[comments_size] = '\0';
- }
-
- c2b_gtf_t gtf;
- gtf.seqname = seqname_str;
- gtf.source = source_str;
- gtf.feature = feature_str;
- gtf.start = start_val;
- gtf.end = end_val;
- gtf.score = score_str;
- gtf.strand = strand_str;
- gtf.frame = frame_str;
- gtf.attributes = attributes_str;
- gtf.comments = comments_str;
+ if (comments_size >= c2b_globals.gtf->element->comments_capacity) {
+ char *comments_resized = NULL;
+ comments_resized = realloc(c2b_globals.gtf->element->comments, comments_size + 1);
+ if (comments_resized) {
+ c2b_globals.gtf->element->comments = comments_resized;
+ c2b_globals.gtf->element->comments_capacity = comments_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize COMMENTS string in GTF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.gtf->element->comments, src + gtf_field_offsets[8] + 1, comments_size);
+ c2b_globals.gtf->element->comments[comments_size] = '\0';
+ }
/*
Fix coordinate indexing, and (if needed) add attribute for zero-length record
*/
- if (gtf.start == gtf.end) {
- gtf.start -= 1;
- ssize_t trailing_semicolon_fudge = (attributes_str[strlen(attributes_str) - 1] == ';') ? 1 : 0;
- memcpy(attributes_str + strlen(attributes_str) - trailing_semicolon_fudge,
+ if (c2b_globals.gtf->element->start == c2b_globals.gtf->element->end) {
+ c2b_globals.gtf->element->start -= 1;
+ ssize_t trailing_semicolon_fudge = (c2b_globals.gtf->element->attributes[strlen(c2b_globals.gtf->element->attributes) - 1] == ';') ? 1 : 0;
+ memcpy(c2b_globals.gtf->element->attributes + strlen(c2b_globals.gtf->element->attributes) - trailing_semicolon_fudge,
c2b_gtf_zero_length_insertion_attribute,
strlen(c2b_gtf_zero_length_insertion_attribute) + 1);
}
else {
- gtf.start -= 1;
+ c2b_globals.gtf->element->start -= 1;
}
/*
@@ -1020,12 +1223,12 @@ c2b_line_convert_gtf_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src,
*/
char *attributes_copy = NULL;
- attributes_copy = malloc(strlen(attributes_str) + 1);
+ attributes_copy = malloc(strlen(c2b_globals.gtf->element->attributes) + 1);
if (!attributes_copy) {
fprintf(stderr, "Error: Could not allocate space for GTF attributes copy\n");
exit(ENOMEM); /* Not enough space (POSIX.1) */
}
- memcpy(attributes_copy, attributes_str, strlen(attributes_str) + 1);
+ memcpy(attributes_copy, c2b_globals.gtf->element->attributes, strlen(c2b_globals.gtf->element->attributes) + 1);
const char *kv_tok;
char *gene_id_str = NULL;
char *transcript_id_str = NULL;
@@ -1052,14 +1255,27 @@ c2b_line_convert_gtf_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src,
exit(ENODATA); /* No data available (POSIX.1) */
}
if ((gtf_id_start && gtf_id_end) && (gtf_id_start != gtf_id_end)) {
- memcpy(c2b_globals.gtf->id, gtf_id_start + 1, gtf_id_end - gtf_id_start - 1);
- c2b_globals.gtf->id[gtf_id_end - gtf_id_start - 1] = '\0';
+ ssize_t id_size = gtf_id_end - gtf_id_start - 1;
+ if (id_size >= c2b_globals.gtf->element->id_capacity) {
+ char *id_resized = NULL;
+ id_resized = realloc(c2b_globals.gtf->element->id, id_size + 1);
+ if (id_resized) {
+ c2b_globals.gtf->element->id = id_resized;
+ c2b_globals.gtf->element->id_capacity = id_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize ID string in GTF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.gtf->element->id, gtf_id_start + 1, id_size);
+ c2b_globals.gtf->element->id[id_size] = '\0';
}
else {
- c2b_globals.gtf->id[0] = '\0';
+ c2b_globals.gtf->element->id[0] = '\0';
}
- if (strlen(c2b_globals.gtf->id) == 0) {
- strcpy(c2b_globals.gtf->id, c2b_gtf_field_placeholder);
+ if (strlen(c2b_globals.gtf->element->id) == 0) {
+ strcpy(c2b_globals.gtf->element->id, c2b_gtf_field_placeholder);
}
gene_id_value_defined = kTrue;
}
@@ -1073,23 +1289,22 @@ c2b_line_convert_gtf_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src,
exit(ENODATA); /* No data available (POSIX.1) */
}
free(attributes_copy), attributes_copy = NULL;
- gtf.id = c2b_globals.gtf->id;
/*
Convert GTF struct to BED string and copy it to destination
*/
- c2b_line_convert_gtf_to_bed(gtf, dest, dest_size);
+ c2b_line_convert_gtf_ptr_to_bed(c2b_globals.gtf->element, dest, dest_size);
c2b_globals.gtf->line_count++;
}
static inline void
-c2b_line_convert_gtf_to_bed(c2b_gtf_t g, char *dest_line, ssize_t *dest_size)
+c2b_line_convert_gtf_ptr_to_bed(c2b_gtf_t *g, char *dest_line, ssize_t *dest_size)
{
/*
For GTF-formatted data, we use the mapping provided by BEDOPS convention described at:
- http://bedops.readthedocs.org/en/latest/content/reference/file-management/conversion/gtf2bed.html
+ http://bedops.readthedocs.io/en/latest/content/reference/file-management/conversion/gtf2bed.html
GTF field BED column index BED field
-------------------------------------------------------------------------
@@ -1116,7 +1331,7 @@ c2b_line_convert_gtf_to_bed(c2b_gtf_t g, char *dest_line, ssize_t *dest_size)
comments 11 -
*/
- if (strlen(g.comments) == 0) {
+ if (strlen(g->comments) == 0) {
*dest_size += sprintf(dest_line + *dest_size,
"%s\t" \
"%" PRIu64 "\t" \
@@ -1128,16 +1343,16 @@ c2b_line_convert_gtf_to_bed(c2b_gtf_t g, char *dest_line, ssize_t *dest_size)
"%s\t" \
"%s\t" \
"%s\n",
- g.seqname,
- g.start,
- g.end,
- g.id,
- g.score,
- g.strand,
- g.source,
- g.feature,
- g.frame,
- g.attributes);
+ g->seqname,
+ g->start,
+ g->end,
+ g->id,
+ g->score,
+ g->strand,
+ g->source,
+ g->feature,
+ g->frame,
+ g->attributes);
}
else {
*dest_size += sprintf(dest_line + *dest_size,
@@ -1152,18 +1367,110 @@ c2b_line_convert_gtf_to_bed(c2b_gtf_t g, char *dest_line, ssize_t *dest_size)
"%s\t" \
"%s\t" \
"%s\n",
- g.seqname,
- g.start,
- g.end,
- g.id,
- g.score,
- g.strand,
- g.source,
- g.feature,
- g.frame,
- g.attributes,
- g.comments);
+ g->seqname,
+ g->start,
+ g->end,
+ g->id,
+ g->score,
+ g->strand,
+ g->source,
+ g->feature,
+ g->frame,
+ g->attributes,
+ g->comments);
+ }
+}
+
+static void
+c2b_gff_init_element(c2b_gff_t **e)
+{
+ *e = malloc(sizeof(c2b_gff_t));
+ if (!*e) {
+ fprintf(stderr, "Error: Could not allocate space for GFF element pointer\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+
+ (*e)->seqid = NULL, (*e)->seqid = malloc(C2B_GFF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->seqid)));
+ if (!(*e)->seqid) {
+ fprintf(stderr, "Error: Could not allocate space for GFF element seqid malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->seqid_capacity = C2B_GFF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->source = NULL, (*e)->source = malloc(C2B_GFF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->source)));
+ if (!(*e)->source) {
+ fprintf(stderr, "Error: Could not allocate space for GFF element source malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
}
+ (*e)->source_capacity = C2B_GFF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->type = NULL, (*e)->type = malloc(C2B_GFF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->type)));
+ if (!(*e)->type) {
+ fprintf(stderr, "Error: Could not allocate space for GFF element type malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->type_capacity = C2B_GFF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->start = 0;
+ (*e)->end = 0;
+
+ (*e)->score = NULL, (*e)->score = malloc(C2B_GFF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->score)));
+ if (!(*e)->score) {
+ fprintf(stderr, "Error: Could not allocate space for GFF element score malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->score_capacity = C2B_GFF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->strand = NULL, (*e)->strand = malloc(C2B_GFF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->strand)));
+ if (!(*e)->strand) {
+ fprintf(stderr, "Error: Could not allocate space for GFF element strand malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->strand_capacity = C2B_GFF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->phase = NULL, (*e)->phase = malloc(C2B_GFF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->phase)));
+ if (!(*e)->phase) {
+ fprintf(stderr, "Error: Could not allocate space for GFF element phase malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->phase_capacity = C2B_GFF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->attributes = NULL, (*e)->attributes = malloc(C2B_GFF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->attributes)));
+ if (!(*e)->attributes) {
+ fprintf(stderr, "Error: Could not allocate space for GFF element attributes malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->attributes_capacity = C2B_GFF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->id = NULL, (*e)->id = malloc(C2B_GFF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->id)));
+ if (!(*e)->id) {
+ fprintf(stderr, "Error: Could not allocate space for GFF element id malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->id_capacity = C2B_GFF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+}
+
+static void
+c2b_gff_delete_element(c2b_gff_t *e)
+{
+ if (e->seqid) { free(e->seqid), e->seqid = NULL; }
+ if (e->source) { free(e->source), e->source = NULL; }
+ if (e->type) { free(e->type), e->type = NULL; }
+ if (e->score) { free(e->score), e->score = NULL; }
+ if (e->strand) { free(e->strand), e->strand = NULL; }
+ if (e->phase) { free(e->phase), e->phase = NULL; }
+ if (e->attributes) { free(e->attributes), e->attributes = NULL; }
+ if (e->id) { free(e->id), e->id = NULL; }
+ if (e) { free(e), e = NULL; }
}
static void
@@ -1233,85 +1540,165 @@ c2b_line_convert_gff_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src,
}
/* 0 - seqid */
- char seqid_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t seqid_size = gff_field_offsets[0];
- memcpy(seqid_str, src, seqid_size);
- seqid_str[seqid_size] = '\0';
+ if (seqid_size >= c2b_globals.gff->element->seqid_capacity) {
+ char *seqid_resized = NULL;
+ seqid_resized = realloc(c2b_globals.gff->element->seqid, seqid_size + 1);
+ if (seqid_resized) {
+ c2b_globals.gff->element->seqid = seqid_resized;
+ c2b_globals.gff->element->seqid_capacity = seqid_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize SEQID string in GFF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.gff->element->seqid, src, seqid_size);
+ c2b_globals.gff->element->seqid[seqid_size] = '\0';
/* 1 - source */
- char source_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t source_size = gff_field_offsets[1] - gff_field_offsets[0] - 1;
- memcpy(source_str, src + gff_field_offsets[0] + 1, source_size);
- source_str[source_size] = '\0';
+ if (source_size >= c2b_globals.gff->element->source_capacity) {
+ char *source_resized = NULL;
+ source_resized = realloc(c2b_globals.gff->element->source, source_size + 1);
+ if (source_resized) {
+ c2b_globals.gff->element->source = source_resized;
+ c2b_globals.gff->element->source_capacity = source_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize SOURCE string in GFF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.gff->element->source, src + gff_field_offsets[0] + 1, source_size);
+ c2b_globals.gff->element->source[source_size] = '\0';
/* 2 - type */
- char type_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t type_size = gff_field_offsets[2] - gff_field_offsets[1] - 1;
- memcpy(type_str, src + gff_field_offsets[1] + 1, type_size);
- type_str[type_size] = '\0';
+ if (type_size >= c2b_globals.gff->element->type_capacity) {
+ char *type_resized = NULL;
+ type_resized = realloc(c2b_globals.gff->element->type, type_size + 1);
+ if (type_resized) {
+ c2b_globals.gff->element->type = type_resized;
+ c2b_globals.gff->element->type_capacity = type_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize TYPE string in GFF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.gff->element->type, src + gff_field_offsets[1] + 1, type_size);
+ c2b_globals.gff->element->type[type_size] = '\0';
/* 3 - start */
char start_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t start_size = gff_field_offsets[3] - gff_field_offsets[2] - 1;
memcpy(start_str, src + gff_field_offsets[2] + 1, start_size);
start_str[start_size] = '\0';
- uint64_t start_val = strtoull(start_str, NULL, 10);
+ c2b_globals.gff->element->start = strtoull(start_str, NULL, 10);
/* 4 - end */
char end_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t end_size = gff_field_offsets[4] - gff_field_offsets[3] - 1;
memcpy(end_str, src + gff_field_offsets[3] + 1, end_size);
end_str[end_size] = '\0';
- uint64_t end_val = strtoull(end_str, NULL, 10);
+ c2b_globals.gff->element->end = strtoull(end_str, NULL, 10);
/* 5 - score */
- char score_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t score_size = gff_field_offsets[5] - gff_field_offsets[4] - 1;
- memcpy(score_str, src + gff_field_offsets[4] + 1, score_size);
- score_str[score_size] = '\0';
+ if (score_size >= c2b_globals.gff->element->score_capacity) {
+ char *score_resized = NULL;
+ score_resized = realloc(c2b_globals.gff->element->score, score_size + 1);
+ if (score_resized) {
+ c2b_globals.gff->element->score = score_resized;
+ c2b_globals.gff->element->score_capacity = score_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize SCORE string in GFF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.gff->element->score, src + gff_field_offsets[4] + 1, score_size);
+ c2b_globals.gff->element->score[score_size] = '\0';
/* 6 - strand */
- char strand_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t strand_size = gff_field_offsets[6] - gff_field_offsets[5] - 1;
- memcpy(strand_str, src + gff_field_offsets[5] + 1, strand_size);
- strand_str[strand_size] = '\0';
+ if (strand_size >= c2b_globals.gff->element->strand_capacity) {
+ char *strand_resized = NULL;
+ strand_resized = realloc(c2b_globals.gff->element->strand, strand_size + 1);
+ if (strand_resized) {
+ c2b_globals.gff->element->strand = strand_resized;
+ c2b_globals.gff->element->strand_capacity = strand_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize STRAND string in GFF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.gff->element->strand, src + gff_field_offsets[5] + 1, strand_size);
+ c2b_globals.gff->element->strand[strand_size] = '\0';
/* 7 - phase */
- char phase_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t phase_size = gff_field_offsets[7] - gff_field_offsets[6] - 1;
- memcpy(phase_str, src + gff_field_offsets[6] + 1, phase_size);
- phase_str[phase_size] = '\0';
+ if (phase_size >= c2b_globals.gff->element->phase_capacity) {
+ char *phase_resized = NULL;
+ phase_resized = realloc(c2b_globals.gff->element->phase, phase_size + 1);
+ if (phase_resized) {
+ c2b_globals.gff->element->phase = phase_resized;
+ c2b_globals.gff->element->phase_capacity = phase_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize PHASE string in GFF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.gff->element->phase, src + gff_field_offsets[6] + 1, phase_size);
+ c2b_globals.gff->element->phase[phase_size] = '\0';
/* 8 - attributes */
- char attributes_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t attributes_size = gff_field_offsets[8] - gff_field_offsets[7] - 1;
- memcpy(attributes_str, src + gff_field_offsets[7] + 1, attributes_size);
- attributes_str[attributes_size] = '\0';
-
- c2b_gff_t gff;
- gff.seqid = seqid_str;
- gff.source = source_str;
- gff.type = type_str;
- gff.start = start_val;
- gff.end = end_val;
- gff.score = score_str;
- gff.strand = strand_str;
- gff.phase = phase_str;
- gff.attributes = attributes_str;
+ if (attributes_size >= c2b_globals.gff->element->attributes_capacity) {
+ char *attributes_resized = NULL;
+ attributes_resized = realloc(c2b_globals.gff->element->attributes, attributes_size * 2);
+ if (attributes_resized) {
+ c2b_globals.gff->element->attributes = attributes_resized;
+ c2b_globals.gff->element->attributes_capacity = attributes_size * 2;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize ATTRIBUTES string in GFF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.gff->element->attributes, src + gff_field_offsets[7] + 1, attributes_size);
+ c2b_globals.gff->element->attributes[attributes_size] = '\0';
/*
Fix coordinate indexing, and (if needed) add attribute for zero-length record
*/
- if (gff.start == gff.end) {
- gff.start -= 1;
- ssize_t trailing_semicolon_fudge = (attributes_str[strlen(attributes_str) - 1] == ';') ? 1 : 0;
- memcpy(attributes_str + strlen(attributes_str) - trailing_semicolon_fudge,
+ if (c2b_globals.gff->element->start == c2b_globals.gff->element->end) {
+ c2b_globals.gff->element->start -= 1;
+ ssize_t trailing_semicolon_fudge = (c2b_globals.gff->element->attributes[strlen(c2b_globals.gff->element->attributes) - 1] == ';') ? 1 : 0;
+ ssize_t new_attributes_size_with_zlia = strlen(c2b_globals.gff->element->attributes) + strlen(c2b_gff_zero_length_insertion_attribute);
+ if (new_attributes_size_with_zlia >= c2b_globals.gff->element->attributes_capacity) {
+ char *attributes_resized_for_zlia = NULL;
+ attributes_resized_for_zlia = realloc(c2b_globals.gff->element->attributes, new_attributes_size_with_zlia + 1);
+ if (attributes_resized_for_zlia) {
+ c2b_globals.gff->element->attributes = attributes_resized_for_zlia;
+ c2b_globals.gff->element->attributes_capacity = new_attributes_size_with_zlia + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize ATTRIBUTES string (for ZLIA) in GFF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.gff->element->attributes + strlen(c2b_globals.gff->element->attributes) - trailing_semicolon_fudge,
c2b_gff_zero_length_insertion_attribute,
strlen(c2b_gff_zero_length_insertion_attribute) + 1);
+ c2b_globals.gff->element->attributes[new_attributes_size_with_zlia] = '\0';
}
else {
- gff.start -= 1;
+ c2b_globals.gff->element->start -= 1;
}
/*
@@ -1319,41 +1706,54 @@ c2b_line_convert_gff_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src,
*/
char *attributes_copy = NULL;
- attributes_copy = malloc(strlen(attributes_str) + 1);
+ attributes_copy = malloc(strlen(c2b_globals.gff->element->attributes) + 1);
if (!attributes_copy) {
fprintf(stderr, "Error: Could not allocate space for GFF attributes copy\n");
exit(ENOMEM); /* Not enough space (POSIX.1) */
}
- memcpy(attributes_copy, attributes_str, strlen(attributes_str) + 1);
+ memcpy(attributes_copy, c2b_globals.gff->element->attributes, strlen(c2b_globals.gff->element->attributes) + 1);
const char *kv_tok;
const char *gff_id_prefix = "ID=";
const char *gff_null_id = ".";
char *id_str = NULL;
- memcpy(c2b_globals.gff->id, gff_null_id, strlen(gff_null_id) + 1);
+ memcpy(c2b_globals.gff->element->id, gff_null_id, strlen(gff_null_id) + 1);
+ c2b_globals.gff->element->id[strlen(gff_null_id)] = '\0';
while ((kv_tok = c2b_strsep(&attributes_copy, ";")) != NULL) {
id_str = strstr(kv_tok, gff_id_prefix);
if (id_str) {
- memcpy(c2b_globals.gff->id, kv_tok + strlen(gff_id_prefix), strlen(kv_tok + strlen(gff_id_prefix)) + 1);
- c2b_globals.gff->id[strlen(kv_tok + strlen(gff_id_prefix)) + 1] = '\0';
+ ssize_t id_size = strlen(id_str);
+ if (id_size >= c2b_globals.gff->element->id_capacity) {
+ char *id_resized = NULL;
+ id_resized = realloc(c2b_globals.gff->element->id, id_size + 1);
+ if (id_resized) {
+ c2b_globals.gff->element->id = id_resized;
+ c2b_globals.gff->element->id_capacity = id_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize ID string in GFF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.gff->element->id, kv_tok + strlen(gff_id_prefix), strlen(kv_tok + strlen(gff_id_prefix)) + 1);
+ c2b_globals.gff->element->id[strlen(kv_tok + strlen(gff_id_prefix)) + 1] = '\0';
}
}
free(attributes_copy), attributes_copy = NULL;
- gff.id = c2b_globals.gff->id;
/*
Convert GFF struct to BED string and copy it to destination
*/
- c2b_line_convert_gff_to_bed(gff, dest, dest_size);
+ c2b_line_convert_gff_ptr_to_bed(c2b_globals.gff->element, dest, dest_size);
}
static inline void
-c2b_line_convert_gff_to_bed(c2b_gff_t g, char *dest_line, ssize_t *dest_size)
+c2b_line_convert_gff_ptr_to_bed(c2b_gff_t *g, char *dest_line, ssize_t *dest_size)
{
/*
For GFF- and GVF-formatted data, we use the mapping provided by BEDOPS convention described at:
- http://bedops.readthedocs.org/en/latest/content/reference/file-management/conversion/gff2bed.html
+ http://bedops.readthedocs.io/en/latest/content/reference/file-management/conversion/gff2bed.html
GFF field BED column index BED field
-------------------------------------------------------------------------
@@ -1385,16 +1785,16 @@ c2b_line_convert_gff_to_bed(c2b_gff_t g, char *dest_line, ssize_t *dest_size)
"%s\t" \
"%s\t" \
"%s\n",
- g.seqid,
- g.start,
- g.end,
- g.id,
- g.score,
- g.strand,
- g.source,
- g.type,
- g.phase,
- g.attributes);
+ g->seqid,
+ g->start,
+ g->end,
+ g->id,
+ g->score,
+ g->strand,
+ g->source,
+ g->type,
+ g->phase,
+ g->attributes);
}
static void
@@ -1825,7 +2225,7 @@ c2b_line_convert_psl_to_bed(c2b_psl_t p, char *dest_line, ssize_t *dest_size)
/*
For PSL-formatted data, we use the mapping provided by BEDOPS convention described at:
- http://bedops.readthedocs.org/en/latest/content/reference/file-management/conversion/psl2bed.html
+ http://bedops.readthedocs.io/en/latest/content/reference/file-management/conversion/psl2bed.html
PSL field BED column index BED field
-------------------------------------------------------------------------
@@ -1903,6 +2303,149 @@ c2b_line_convert_psl_to_bed(c2b_psl_t p, char *dest_line, ssize_t *dest_size)
}
static void
+c2b_rmsk_init_element(c2b_rmsk_t **e)
+{
+ *e = malloc(sizeof(c2b_rmsk_t));
+ if (!*e) {
+ fprintf(stderr, "Error: Could not allocate space for RMSK element pointer\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+
+ (*e)->sw_score = NULL, (*e)->sw_score = malloc(C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->sw_score)));
+ if (!(*e)->sw_score) {
+ fprintf(stderr, "Error: Could not allocate space for RMSK element sw_score malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->sw_score_capacity = C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->perc_div = NULL, (*e)->perc_div = malloc(C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->perc_div)));
+ if (!(*e)->perc_div) {
+ fprintf(stderr, "Error: Could not allocate space for RMSK element perc_div malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->perc_div_capacity = C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->perc_deleted = NULL, (*e)->perc_deleted = malloc(C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->perc_deleted)));
+ if (!(*e)->perc_deleted) {
+ fprintf(stderr, "Error: Could not allocate space for RMSK element perc_deleted malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->perc_deleted_capacity = C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->perc_inserted = NULL, (*e)->perc_inserted = malloc(C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->perc_inserted)));
+ if (!(*e)->perc_inserted) {
+ fprintf(stderr, "Error: Could not allocate space for RMSK element perc_inserted malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->perc_inserted_capacity = C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->query_seq = NULL, (*e)->query_seq = malloc(C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->query_seq)));
+ if (!(*e)->query_seq) {
+ fprintf(stderr, "Error: Could not allocate space for RMSK element query_seq malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->query_seq_capacity = C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->bases_past_match = NULL, (*e)->bases_past_match = malloc(C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->bases_past_match)));
+ if (!(*e)->bases_past_match) {
+ fprintf(stderr, "Error: Could not allocate space for RMSK element bases_past_match malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->bases_past_match_capacity = C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->strand = NULL, (*e)->strand = malloc(C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->strand)));
+ if (!(*e)->strand) {
+ fprintf(stderr, "Error: Could not allocate space for RMSK element strand malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->strand_capacity = C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->repeat_name = NULL, (*e)->repeat_name = malloc(C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->repeat_name)));
+ if (!(*e)->repeat_name) {
+ fprintf(stderr, "Error: Could not allocate space for RMSK element repeat_name malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->repeat_name_capacity = C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->repeat_class = NULL, (*e)->repeat_class = malloc(C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->repeat_class)));
+ if (!(*e)->repeat_class) {
+ fprintf(stderr, "Error: Could not allocate space for RMSK element repeat_class malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->repeat_class_capacity = C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->bases_before_match_comp = NULL, (*e)->bases_before_match_comp = malloc(C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->bases_before_match_comp)));
+ if (!(*e)->bases_before_match_comp) {
+ fprintf(stderr, "Error: Could not allocate space for RMSK element bases_before_match_comp malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->bases_before_match_comp_capacity = C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->match_start = NULL, (*e)->match_start = malloc(C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->match_start)));
+ if (!(*e)->match_start) {
+ fprintf(stderr, "Error: Could not allocate space for RMSK element match_start malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->match_start_capacity = C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->match_end = NULL, (*e)->match_end = malloc(C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->match_end)));
+ if (!(*e)->match_end) {
+ fprintf(stderr, "Error: Could not allocate space for RMSK element match_end malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->match_end_capacity = C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->unique_id = NULL, (*e)->unique_id = malloc(C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->unique_id)));
+ if (!(*e)->unique_id) {
+ fprintf(stderr, "Error: Could not allocate space for RMSK element unique_id malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->unique_id_capacity = C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->higher_score_match = NULL, (*e)->higher_score_match = malloc(C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->higher_score_match)));
+ if (!(*e)->higher_score_match) {
+ fprintf(stderr, "Error: Could not allocate space for RMSK element higher_score_match malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->higher_score_match_capacity = C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+}
+
+static void
+c2b_rmsk_delete_element(c2b_rmsk_t *e)
+{
+ if (e->sw_score) { free(e->sw_score), e->sw_score = NULL; }
+ if (e->perc_div) { free(e->perc_div), e->perc_div = NULL; }
+ if (e->perc_deleted) { free(e->perc_deleted), e->perc_deleted = NULL; }
+ if (e->perc_inserted) { free(e->perc_inserted), e->perc_inserted = NULL; }
+ if (e->query_seq) { free(e->query_seq), e->query_seq = NULL; }
+ if (e->bases_past_match) { free(e->bases_past_match), e->bases_past_match = NULL; }
+ if (e->strand) { free(e->strand), e->strand = NULL; }
+ if (e->repeat_name) { free(e->repeat_name), e->repeat_name = NULL; }
+ if (e->repeat_class) { free(e->repeat_class), e->repeat_class = NULL; }
+ if (e->bases_before_match_comp) { free(e->bases_before_match_comp), e->bases_before_match_comp = NULL; }
+ if (e->match_start) { free(e->match_start), e->match_start = NULL; }
+ if (e->match_end) { free(e->match_end), e->match_end = NULL; }
+ if (e->unique_id) { free(e->unique_id), e->unique_id = NULL; }
+ if (e->higher_score_match) { free(e->higher_score_match), e->higher_score_match = NULL; }
+ if (e) { free(e), e = NULL; }
+}
+
+static void
c2b_line_convert_rmsk_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src, ssize_t src_size)
{
/*
@@ -1916,6 +2459,12 @@ c2b_line_convert_rmsk_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src,
int rmsk_field_end_idx = 0;
ssize_t current_src_posn = 0;
+ /* skip blank lines */
+ if (src_size == 0) {
+ c2b_globals.rmsk->line++;
+ return;
+ }
+
while (current_src_posn < src_size) {
/* within bounds */
if (((current_src_posn + 1) < src_size) && (c2b_globals.rmsk->line >= c2b_rmsk_header_line_count)) {
@@ -1999,63 +2548,123 @@ c2b_line_convert_rmsk_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src,
}
/* 0 - Smith-Waterman score of the match */
- char sw_score_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t sw_score_start = rmsk_field_start_offsets[0];
ssize_t sw_score_end = rmsk_field_end_offsets[0];
ssize_t sw_score_size = sw_score_end - sw_score_start;
- memcpy(sw_score_str, src + sw_score_start, sw_score_size);
- sw_score_str[sw_score_size] = '\0';
+ if (sw_score_size >= c2b_globals.rmsk->element->sw_score_capacity) {
+ char *sw_score_resized = NULL;
+ sw_score_resized = realloc(c2b_globals.rmsk->element->sw_score, sw_score_size + 1);
+ if (sw_score_resized) {
+ c2b_globals.rmsk->element->sw_score = sw_score_resized;
+ c2b_globals.rmsk->element->sw_score_capacity = sw_score_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize SW_SCORE string in RMSK element struct\n");
+ exit(ENOMEM);
+ }
+
+ }
+ memcpy(c2b_globals.rmsk->element->sw_score, src + sw_score_start, sw_score_size);
+ c2b_globals.rmsk->element->sw_score[sw_score_size] = '\0';
#ifdef DEBUG
- fprintf(stderr, "sw_score_str [%s]\n", sw_score_str);
+ fprintf(stderr, "sw_score [%s]\n", c2b_globals.rmsk->element->sw_score);
#endif
/* 1 - Percent, divergence = mismatches / (matches + mismatches) */
- char perc_div_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t perc_div_start = rmsk_field_start_offsets[1];
ssize_t perc_div_end = rmsk_field_end_offsets[1];
ssize_t perc_div_size = perc_div_end - perc_div_start;
- memcpy(perc_div_str, src + perc_div_start, perc_div_size);
- perc_div_str[perc_div_size] = '\0';
+ if (perc_div_size >= c2b_globals.rmsk->element->perc_div_capacity) {
+ char *perc_div_resized = NULL;
+ perc_div_resized = realloc(c2b_globals.rmsk->element->perc_div, perc_div_size + 1);
+ if (perc_div_resized) {
+ c2b_globals.rmsk->element->perc_div = perc_div_resized;
+ c2b_globals.rmsk->element->perc_div_capacity = perc_div_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize PERC_DIV string in RMSK element struct\n");
+ exit(ENOMEM);
+ }
+
+ }
+ memcpy(c2b_globals.rmsk->element->perc_div, src + perc_div_start, perc_div_size);
+ c2b_globals.rmsk->element->perc_div[perc_div_size] = '\0';
#ifdef DEBUG
- fprintf(stderr, "perc_div_str [%s]\n", perc_div_str);
+ fprintf(stderr, "perc_div [%s]\n", c2b_globals.rmsk->element->perc_div);
#endif
/* 2 - Percent, bases opposite a gap in the query sequence = deleted bp */
- char perc_deleted_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t perc_deleted_start = rmsk_field_start_offsets[2];
ssize_t perc_deleted_end = rmsk_field_end_offsets[2];
ssize_t perc_deleted_size = perc_deleted_end - perc_deleted_start;
- memcpy(perc_deleted_str, src + perc_deleted_start, perc_deleted_size);
- perc_deleted_str[perc_deleted_size] = '\0';
+ if (perc_deleted_size >= c2b_globals.rmsk->element->perc_deleted_capacity) {
+ char *perc_deleted_resized = NULL;
+ perc_deleted_resized = realloc(c2b_globals.rmsk->element->perc_deleted, perc_deleted_size + 1);
+ if (perc_deleted_resized) {
+ c2b_globals.rmsk->element->perc_deleted = perc_deleted_resized;
+ c2b_globals.rmsk->element->perc_deleted_capacity = perc_deleted_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize PERC_DELETED string in RMSK element struct\n");
+ exit(ENOMEM);
+ }
+
+ }
+ memcpy(c2b_globals.rmsk->element->perc_deleted, src + perc_deleted_start, perc_deleted_size);
+ c2b_globals.rmsk->element->perc_deleted[perc_deleted_size] = '\0';
#ifdef DEBUG
- fprintf(stderr, "perc_deleted_str [%s]\n", perc_deleted_str);
+ fprintf(stderr, "perc_deleted [%s]\n", c2b_globals.rmsk->element->perc_deleted);
#endif
/* 3 - Percent, bases opposite a gap in the repeat consensus = inserted bp */
- char perc_inserted_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t perc_inserted_start = rmsk_field_start_offsets[3];
ssize_t perc_inserted_end = rmsk_field_end_offsets[3];
ssize_t perc_inserted_size = perc_inserted_end - perc_inserted_start;
- memcpy(perc_inserted_str, src + perc_inserted_start, perc_inserted_size);
- perc_inserted_str[perc_inserted_size] = '\0';
+ if (perc_inserted_size >= c2b_globals.rmsk->element->perc_inserted_capacity) {
+ char *perc_inserted_resized = NULL;
+ perc_inserted_resized = realloc(c2b_globals.rmsk->element->perc_inserted, perc_inserted_size + 1);
+ if (perc_inserted_resized) {
+ c2b_globals.rmsk->element->perc_inserted = perc_inserted_resized;
+ c2b_globals.rmsk->element->perc_inserted_capacity = perc_inserted_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize PERC_INSERTED string in RMSK element struct\n");
+ exit(ENOMEM);
+ }
+
+ }
+ memcpy(c2b_globals.rmsk->element->perc_inserted, src + perc_inserted_start, perc_inserted_size);
+ c2b_globals.rmsk->element->perc_inserted[perc_inserted_size] = '\0';
#ifdef DEBUG
- fprintf(stderr, "perc_inserted_str [%s]\n", perc_inserted_str);
+ fprintf(stderr, "perc_inserted [%s]\n", c2b_globals.rmsk->element->perc_inserted);
#endif
/* 4 - Query sequence */
- char query_seq_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t query_seq_start = rmsk_field_start_offsets[4];
ssize_t query_seq_end = rmsk_field_end_offsets[4];
ssize_t query_seq_size = query_seq_end - query_seq_start;
- memcpy(query_seq_str, src + query_seq_start, query_seq_size);
- query_seq_str[query_seq_size] = '\0';
+ if (query_seq_size >= c2b_globals.rmsk->element->query_seq_capacity) {
+ char *query_seq_resized = NULL;
+ query_seq_resized = realloc(c2b_globals.rmsk->element->query_seq, query_seq_size + 1);
+ if (query_seq_resized) {
+ c2b_globals.rmsk->element->query_seq = query_seq_resized;
+ c2b_globals.rmsk->element->query_seq_capacity = query_seq_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize QUERY_SEQ string in RMSK element struct\n");
+ exit(ENOMEM);
+ }
+
+ }
+ memcpy(c2b_globals.rmsk->element->query_seq, src + query_seq_start, query_seq_size);
+ c2b_globals.rmsk->element->query_seq[query_seq_size] = '\0';
#ifdef DEBUG
- fprintf(stderr, "query_seq_str [%s]\n", query_seq_str);
+ fprintf(stderr, "query_seq [%s]\n", c2b_globals.rmsk->element->query_seq);
#endif
/* 5 - Query start (1-indexed) */
@@ -2065,7 +2674,14 @@ c2b_line_convert_rmsk_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src,
ssize_t query_start_size = query_start_end - query_start_start;
memcpy(query_start_str, src + query_start_start, query_start_size);
query_start_str[query_start_size] = '\0';
- uint64_t query_start_val = strtoull(query_start_str, NULL, 10);
+ c2b_globals.rmsk->element->query_start = strtoull(query_start_str, NULL, 10);
+ if (errno == ERANGE) {
+ fprintf(stderr, "Error: Could not convert QUERY_START string [%s] to integer (check input)\n", query_start_str);
+ exit(ERANGE);
+ }
+
+ /* subtract to make 0-indexed */
+ c2b_globals.rmsk->element->query_start--;
#ifdef DEBUG
fprintf(stderr, "query_start_str [%s]\n", query_start_str);
@@ -2078,156 +2694,250 @@ c2b_line_convert_rmsk_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src,
ssize_t query_end_size = query_end_end - query_end_start;
memcpy(query_end_str, src + query_end_start, query_end_size);
query_end_str[query_end_size] = '\0';
- uint64_t query_end_val = strtoull(query_end_str, NULL, 10);
+ c2b_globals.rmsk->element->query_end = strtoull(query_end_str, NULL, 10);
+ if (errno == ERANGE) {
+ fprintf(stderr, "Error: Could not convert QUERY_END string [%s] to integer (check input)\n", query_end_str);
+ exit(ERANGE);
+ }
#ifdef DEBUG
fprintf(stderr, "query_end_str [%s]\n", query_end_str);
#endif
/* 7 - Bases in query sequence past the ending position of match */
- char bases_past_match_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t bases_past_match_start = rmsk_field_start_offsets[7];
ssize_t bases_past_match_end = rmsk_field_end_offsets[7];
ssize_t bases_past_match_size = bases_past_match_end - bases_past_match_start;
- memcpy(bases_past_match_str, src + bases_past_match_start, bases_past_match_size);
- bases_past_match_str[bases_past_match_size] = '\0';
+ if (bases_past_match_size >= c2b_globals.rmsk->element->bases_past_match_capacity) {
+ char *bases_past_match_resized = NULL;
+ bases_past_match_resized = realloc(c2b_globals.rmsk->element->bases_past_match, bases_past_match_size + 1);
+ if (bases_past_match_resized) {
+ c2b_globals.rmsk->element->bases_past_match = bases_past_match_resized;
+ c2b_globals.rmsk->element->bases_past_match_capacity = bases_past_match_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize BASES_PAST_MATCH string in RMSK element struct\n");
+ exit(ENOMEM);
+ }
+
+ }
+ memcpy(c2b_globals.rmsk->element->bases_past_match, src + bases_past_match_start, bases_past_match_size);
+ c2b_globals.rmsk->element->bases_past_match[bases_past_match_size] = '\0';
#ifdef DEBUG
- fprintf(stderr, "bases_past_match_str [%s]\n", bases_past_match_str);
+ fprintf(stderr, "bases_past_match [%s]\n", c2b_globals.rmsk->element->bases_past_match);
#endif
/* 8 - Strand match with repeat consensus sequence (+ = forward, C = complement) */
- char strand_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t strand_start = rmsk_field_start_offsets[8];
ssize_t strand_end = rmsk_field_end_offsets[8];
ssize_t strand_size = strand_end - strand_start;
- memcpy(strand_str, src + strand_start, strand_size);
- strand_str[strand_size] = '\0';
- if (strcmp(strand_str, c2b_rmsk_strand_complement) == 0) {
- memcpy(strand_str, c2b_rmsk_strand_complement_replacement, strlen(c2b_rmsk_strand_complement_replacement) + 1);
+ if (strand_size >= c2b_globals.rmsk->element->strand_capacity) {
+ char *strand_resized = NULL;
+ strand_resized = realloc(c2b_globals.rmsk->element->strand, strand_size + 1);
+ if (strand_resized) {
+ c2b_globals.rmsk->element->strand = strand_resized;
+ c2b_globals.rmsk->element->strand_capacity = strand_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize STRAND string in RMSK element struct\n");
+ exit(ENOMEM);
+ }
+
+ }
+ memcpy(c2b_globals.rmsk->element->strand, src + strand_start, strand_size);
+ c2b_globals.rmsk->element->strand[strand_size] = '\0';
+ if (strcmp(c2b_globals.rmsk->element->strand, c2b_rmsk_strand_complement) == 0) {
+ memcpy(c2b_globals.rmsk->element->strand, c2b_rmsk_strand_complement_replacement, strlen(c2b_rmsk_strand_complement_replacement) + 1);
}
#ifdef DEBUG
- fprintf(stderr, "strand_str [%s]\n", strand_str);
+ fprintf(stderr, "strand [%s]\n", c2b_globals.rmsk->element->strand);
#endif
/* 9 - Matching interspersed repeat name */
- char repeat_name_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t repeat_name_start = rmsk_field_start_offsets[9];
ssize_t repeat_name_end = rmsk_field_end_offsets[9];
ssize_t repeat_name_size = repeat_name_end - repeat_name_start;
- memcpy(repeat_name_str, src + repeat_name_start, repeat_name_size);
- repeat_name_str[repeat_name_size] = '\0';
+ if (repeat_name_size >= c2b_globals.rmsk->element->repeat_name_capacity) {
+ char *repeat_name_resized = NULL;
+ repeat_name_resized = realloc(c2b_globals.rmsk->element->repeat_name, repeat_name_size + 1);
+ if (repeat_name_resized) {
+ c2b_globals.rmsk->element->repeat_name = repeat_name_resized;
+ c2b_globals.rmsk->element->repeat_name_capacity = repeat_name_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize REPEAT_NAME string in RMSK element struct\n");
+ exit(ENOMEM);
+ }
+
+ }
+ memcpy(c2b_globals.rmsk->element->repeat_name, src + repeat_name_start, repeat_name_size);
+ c2b_globals.rmsk->element->repeat_name[repeat_name_size] = '\0';
#ifdef DEBUG
- fprintf(stderr, "repeat_name_str [%s]\n", repeat_name_str);
+ fprintf(stderr, "repeat_name [%s]\n", c2b_globals.rmsk->element->repeat_name);
#endif
/* 10 - Repeat class */
- char repeat_class_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t repeat_class_start = rmsk_field_start_offsets[10];
ssize_t repeat_class_end = rmsk_field_end_offsets[10];
ssize_t repeat_class_size = repeat_class_end - repeat_class_start;
- memcpy(repeat_class_str, src + repeat_class_start, repeat_class_size);
- repeat_class_str[repeat_class_size] = '\0';
+ if (repeat_class_size >= c2b_globals.rmsk->element->repeat_class_capacity) {
+ char *repeat_class_resized = NULL;
+ repeat_class_resized = realloc(c2b_globals.rmsk->element->repeat_class, repeat_class_size + 1);
+ if (repeat_class_resized) {
+ c2b_globals.rmsk->element->repeat_class = repeat_class_resized;
+ c2b_globals.rmsk->element->repeat_class_capacity = repeat_class_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize REPEAT_CLASS string in RMSK element struct\n");
+ exit(ENOMEM);
+ }
+
+ }
+ memcpy(c2b_globals.rmsk->element->repeat_class, src + repeat_class_start, repeat_class_size);
+ c2b_globals.rmsk->element->repeat_class[repeat_class_size] = '\0';
#ifdef DEBUG
- fprintf(stderr, "repeat_class_str [%s]\n", repeat_class_str);
+ fprintf(stderr, "repeat_class [%s]\n", c2b_globals.rmsk->element->repeat_class);
#endif
/* 11 - Bases in (complement of) the repeat consensus sequence, prior to beginning of the match */
- char bases_before_match_comp_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t bases_before_match_comp_start = rmsk_field_start_offsets[11];
ssize_t bases_before_match_comp_end = rmsk_field_end_offsets[11];
ssize_t bases_before_match_comp_size = bases_before_match_comp_end - bases_before_match_comp_start;
- memcpy(bases_before_match_comp_str, src + bases_before_match_comp_start, bases_before_match_comp_size);
- bases_before_match_comp_str[bases_before_match_comp_size] = '\0';
+ if (bases_before_match_comp_size >= c2b_globals.rmsk->element->bases_before_match_comp_capacity) {
+ char *bases_before_match_comp_resized = NULL;
+ bases_before_match_comp_resized = realloc(c2b_globals.rmsk->element->bases_before_match_comp, bases_before_match_comp_size + 1);
+ if (bases_before_match_comp_resized) {
+ c2b_globals.rmsk->element->bases_before_match_comp = bases_before_match_comp_resized;
+ c2b_globals.rmsk->element->bases_before_match_comp_capacity = bases_before_match_comp_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize BASES_BEFORE_MATCH_COMP string in RMSK element struct\n");
+ exit(ENOMEM);
+ }
+
+ }
+ memcpy(c2b_globals.rmsk->element->bases_before_match_comp, src + bases_before_match_comp_start, bases_before_match_comp_size);
+ c2b_globals.rmsk->element->bases_before_match_comp[bases_before_match_comp_size] = '\0';
#ifdef DEBUG
- fprintf(stderr, "bases_before_match_comp_str [%s]\n", bases_before_match_comp_str);
+ fprintf(stderr, "bases_before_match_comp [%s]\n", c2b_globals.rmsk->element->bases_before_match_comp);
#endif
/* 12 - Match start (in repeat consensus sequence) */
- char match_start_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t match_start_start = rmsk_field_start_offsets[12];
ssize_t match_start_end = rmsk_field_end_offsets[12];
ssize_t match_start_size = match_start_end - match_start_start;
- memcpy(match_start_str, src + match_start_start, match_start_size);
- match_start_str[match_start_size] = '\0';
+ if (match_start_size >= c2b_globals.rmsk->element->match_start_capacity) {
+ char *match_start_resized = NULL;
+ match_start_resized = realloc(c2b_globals.rmsk->element->match_start, match_start_size + 1);
+ if (match_start_resized) {
+ c2b_globals.rmsk->element->match_start = match_start_resized;
+ c2b_globals.rmsk->element->match_start_capacity = match_start_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize MATCH_START string in RMSK element struct\n");
+ exit(ENOMEM);
+ }
+
+ }
+ memcpy(c2b_globals.rmsk->element->match_start, src + match_start_start, match_start_size);
+ c2b_globals.rmsk->element->match_start[match_start_size] = '\0';
#ifdef DEBUG
- fprintf(stderr, "match_start_str [%s]\n", match_start_str);
+ fprintf(stderr, "match_start [%s]\n", c2b_globals.rmsk->element->match_start);
#endif
/* 13 - Match end (in repeat consensus sequence) */
- char match_end_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t match_end_start = rmsk_field_start_offsets[13];
ssize_t match_end_end = rmsk_field_end_offsets[13];
ssize_t match_end_size = match_end_end - match_end_start;
- memcpy(match_end_str, src + match_end_start, match_end_size);
- match_end_str[match_end_size] = '\0';
+ if (match_end_size >= c2b_globals.rmsk->element->match_end_capacity) {
+ char *match_end_resized = NULL;
+ match_end_resized = realloc(c2b_globals.rmsk->element->match_end, match_end_size + 1);
+ if (match_end_resized) {
+ c2b_globals.rmsk->element->match_end = match_end_resized;
+ c2b_globals.rmsk->element->match_end_capacity = match_end_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize MATCH_END string in RMSK element struct\n");
+ exit(ENOMEM);
+ }
+
+ }
+ memcpy(c2b_globals.rmsk->element->match_end, src + match_end_start, match_end_size);
+ c2b_globals.rmsk->element->match_end[match_end_size] = '\0';
#ifdef DEBUG
- fprintf(stderr, "match_end_str [%s]\n", match_end_str);
+ fprintf(stderr, "match_end [%s]\n", c2b_globals.rmsk->element->match_end);
#endif
/* 14 - Identifier for individual insertions */
- char unique_id_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t unique_id_start = rmsk_field_start_offsets[14];
ssize_t unique_id_end = rmsk_field_end_offsets[14];
ssize_t unique_id_size = unique_id_end - unique_id_start;
- memcpy(unique_id_str, src + unique_id_start, unique_id_size);
- unique_id_str[unique_id_size] = '\0';
+ if (unique_id_size >= c2b_globals.rmsk->element->unique_id_capacity) {
+ char *unique_id_resized = NULL;
+ unique_id_resized = realloc(c2b_globals.rmsk->element->unique_id, unique_id_size + 1);
+ if (unique_id_resized) {
+ c2b_globals.rmsk->element->unique_id = unique_id_resized;
+ c2b_globals.rmsk->element->unique_id_capacity = unique_id_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize UNIQUE_ID string in RMSK element struct\n");
+ exit(ENOMEM);
+ }
+
+ }
+ memcpy(c2b_globals.rmsk->element->unique_id, src + unique_id_start, unique_id_size);
+ c2b_globals.rmsk->element->unique_id[unique_id_size] = '\0';
#ifdef DEBUG
- fprintf(stderr, "unique_id_str [%s]\n", unique_id_str);
+ fprintf(stderr, "unique_id [%s]\n", c2b_globals.rmsk->element->unique_id);
#endif
/* 15 - Higher-scoring match present (optional) */
- char higher_score_match_str[C2B_MAX_FIELD_LENGTH_VALUE];
- higher_score_match_str[0] = '\0';
-
if ((rmsk_field_start_idx == c2b_rmsk_field_max) && (rmsk_field_end_idx == c2b_rmsk_field_max)) {
ssize_t higher_score_match_start = rmsk_field_start_offsets[15];
ssize_t higher_score_match_end = rmsk_field_end_offsets[15];
ssize_t higher_score_match_size = higher_score_match_end - higher_score_match_start;
- memcpy(higher_score_match_str, src + higher_score_match_start, higher_score_match_size);
- higher_score_match_str[higher_score_match_size] = '\0';
- }
+ if (higher_score_match_size >= c2b_globals.rmsk->element->higher_score_match_capacity) {
+ char *higher_score_match_resized = NULL;
+ higher_score_match_resized = realloc(c2b_globals.rmsk->element->higher_score_match, higher_score_match_size + 1);
+ if (higher_score_match_resized) {
+ c2b_globals.rmsk->element->higher_score_match = higher_score_match_resized;
+ c2b_globals.rmsk->element->higher_score_match_capacity = higher_score_match_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize HIGHER SCORE MATCH string in RMSK element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.rmsk->element->higher_score_match, src + higher_score_match_start, higher_score_match_size);
+ c2b_globals.rmsk->element->higher_score_match[higher_score_match_size] = '\0';
#ifdef DEBUG
- fprintf(stderr, "higher_score_match_str [%s]\n", higher_score_match_str);
+ fprintf(stderr, "higher_score_match [%s]\n", c2b_globals.rmsk->element->higher_score_match);
#endif
+ }
- c2b_rmsk_t rmsk;
- rmsk.sw_score = sw_score_str;
- rmsk.perc_div = perc_div_str;
- rmsk.perc_deleted = perc_deleted_str;
- rmsk.perc_inserted = perc_inserted_str;
- rmsk.query_seq = query_seq_str;
- rmsk.query_start = query_start_val - 1;
- rmsk.query_end = query_end_val;
- rmsk.bases_past_match = bases_past_match_str;
- rmsk.strand = strand_str;
- rmsk.repeat_name = repeat_name_str;
- rmsk.repeat_class = repeat_class_str;
- rmsk.bases_before_match_comp = bases_before_match_comp_str;
- rmsk.match_start = match_start_str;
- rmsk.match_end = match_end_str;
- rmsk.unique_id = unique_id_str;
- rmsk.higher_score_match = higher_score_match_str;
+ c2b_line_convert_rmsk_ptr_to_bed(c2b_globals.rmsk->element, dest, dest_size);
- c2b_line_convert_rmsk_to_bed(rmsk, dest, dest_size);
+ /* after writing a line, reset length of element higher_score_match string */
+ c2b_globals.rmsk->element->higher_score_match[0] = '\0';
}
static inline void
-c2b_line_convert_rmsk_to_bed(c2b_rmsk_t r, char *dest_line, ssize_t *dest_size)
+c2b_line_convert_rmsk_ptr_to_bed(c2b_rmsk_t *r, char *dest_line, ssize_t *dest_size)
{
/*
For RepeatMasker annotation-formatted data, we use the mapping provided by BEDOPS
convention described at:
- http://bedops.readthedocs.org/en/latest/content/reference/file-management/conversion/rmsk2bed.html
+ http://bedops.readthedocs.io/en/latest/content/reference/file-management/conversion/rmsk2bed.html
RepeatMasker field BED column index BED field
-------------------------------------------------------------------------
@@ -2254,7 +2964,7 @@ c2b_line_convert_rmsk_to_bed(c2b_rmsk_t r, char *dest_line, ssize_t *dest_size)
higher_score_match 16 -
*/
- if (strlen(r.higher_score_match) == 0) {
+ if (strlen(r->higher_score_match) == 0) {
*dest_size += sprintf(dest_line + *dest_size,
"%s\t" \
"%" PRIu64 "\t" \
@@ -2271,21 +2981,21 @@ c2b_line_convert_rmsk_to_bed(c2b_rmsk_t r, char *dest_line, ssize_t *dest_size)
"%s\t" \
"%s\t" \
"%s\n",
- r.query_seq,
- r.query_start,
- r.query_end,
- r.repeat_name,
- r.sw_score,
- r.strand,
- r.perc_div,
- r.perc_deleted,
- r.perc_inserted,
- r.bases_past_match,
- r.repeat_class,
- r.bases_before_match_comp,
- r.match_start,
- r.match_end,
- r.unique_id);
+ r->query_seq,
+ r->query_start,
+ r->query_end,
+ r->repeat_name,
+ r->sw_score,
+ r->strand,
+ r->perc_div,
+ r->perc_deleted,
+ r->perc_inserted,
+ r->bases_past_match,
+ r->repeat_class,
+ r->bases_before_match_comp,
+ r->match_start,
+ r->match_end,
+ r->unique_id);
}
else {
*dest_size += sprintf(dest_line + *dest_size,
@@ -2305,22 +3015,22 @@ c2b_line_convert_rmsk_to_bed(c2b_rmsk_t r, char *dest_line, ssize_t *dest_size)
"%s\t" \
"%s\t" \
"%s\n",
- r.query_seq,
- r.query_start,
- r.query_end,
- r.repeat_name,
- r.sw_score,
- r.strand,
- r.perc_div,
- r.perc_deleted,
- r.perc_inserted,
- r.bases_past_match,
- r.repeat_class,
- r.bases_before_match_comp,
- r.match_start,
- r.match_end,
- r.unique_id,
- r.higher_score_match);
+ r->query_seq,
+ r->query_start,
+ r->query_end,
+ r->repeat_name,
+ r->sw_score,
+ r->strand,
+ r->perc_div,
+ r->perc_deleted,
+ r->perc_inserted,
+ r->bases_past_match,
+ r->repeat_class,
+ r->bases_before_match_comp,
+ r->match_start,
+ r->match_end,
+ r->unique_id,
+ r->higher_score_match);
}
}
@@ -2347,8 +3057,8 @@ c2b_line_convert_sam_to_bed_unsorted_without_split_operation(char *dest, ssize_t
}
else {
/* copy header line to destination stream buffer */
- char src_header_line_str[C2B_MAX_LINE_LENGTH_VALUE];
- char dest_header_line_str[C2B_MAX_LINE_LENGTH_VALUE];
+ char src_header_line_str[C2B_MAX_LINE_LENGTH_VALUE] = {0};
+ char dest_header_line_str[C2B_MAX_LINE_LENGTH_VALUE] = {0};
memcpy(src_header_line_str, src, src_size);
src_header_line_str[src_size] = '\0';
sprintf(dest_header_line_str, "%s\t%u\t%u\t%s\n", c2b_header_chr_name, c2b_globals.header_line_idx, (c2b_globals.header_line_idx + 1), src_header_line_str);
@@ -2387,10 +3097,21 @@ c2b_line_convert_sam_to_bed_unsorted_without_split_operation(char *dest, ssize_t
*/
ssize_t cigar_size = sam_field_offsets[5] - sam_field_offsets[4];
- char cigar_str[C2B_MAX_FIELD_LENGTH_VALUE];
- memcpy(cigar_str, src + sam_field_offsets[4] + 1, cigar_size - 1);
- cigar_str[cigar_size - 1] = '\0';
- c2b_sam_cigar_str_to_ops(cigar_str);
+ if (cigar_size >= c2b_globals.sam->element->cigar_capacity) {
+ char *cigar_resized = NULL;
+ cigar_resized = realloc(c2b_globals.sam->element->cigar, cigar_size + 1);
+ if (cigar_resized) {
+ c2b_globals.sam->element->cigar = cigar_resized;
+ c2b_globals.sam->element->cigar_capacity = cigar_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize CIGAR string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->cigar, src + sam_field_offsets[4] + 1, cigar_size - 1);
+ c2b_globals.sam->element->cigar[cigar_size - 1] = '\0';
+ c2b_sam_cigar_str_to_ops(c2b_globals.sam->element->cigar);
#ifdef DEBUG
c2b_sam_debug_cigar_ops(c2b_globals.sam->cigar);
#endif
@@ -2405,11 +3126,11 @@ c2b_line_convert_sam_to_bed_unsorted_without_split_operation(char *dest, ssize_t
*/
ssize_t flag_size = sam_field_offsets[1] - sam_field_offsets[0];
- char flag_src_str[C2B_MAX_FIELD_LENGTH_VALUE];
+ char flag_src_str[C2B_MAX_FIELD_LENGTH_VALUE] = {0};
memcpy(flag_src_str, src + sam_field_offsets[0] + 1, flag_size);
flag_src_str[flag_size] = '\0';
- int flag_val = (int) strtol(flag_src_str, NULL, 10);
- boolean is_mapped = (boolean) !(4 & flag_val);
+ c2b_globals.sam->element->flag = (int) strtol(flag_src_str, NULL, 10);
+ boolean is_mapped = (boolean) !(4 & c2b_globals.sam->element->flag);
if ((!is_mapped) && (!c2b_globals.all_reads_flag))
return;
@@ -2418,84 +3139,212 @@ c2b_line_convert_sam_to_bed_unsorted_without_split_operation(char *dest, ssize_t
*/
/* RNAME */
- char rname_str[C2B_MAX_FIELD_LENGTH_VALUE];
if (is_mapped) {
ssize_t rname_size = sam_field_offsets[2] - sam_field_offsets[1] - 1;
- memcpy(rname_str, src + sam_field_offsets[1] + 1, rname_size);
- rname_str[rname_size] = '\0';
+ if (rname_size >= c2b_globals.sam->element->rname_capacity) {
+ char *rname_resized = NULL;
+ rname_resized = realloc(c2b_globals.sam->element->rname, rname_size + 1);
+ if (rname_resized) {
+ c2b_globals.sam->element->rname = rname_resized;
+ c2b_globals.sam->element->rname_capacity = rname_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize RNAME string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->rname, src + sam_field_offsets[1] + 1, rname_size);
+ c2b_globals.sam->element->rname[rname_size] = '\0';
}
else {
- char unmapped_read_chr_str[C2B_MAX_FIELD_LENGTH_VALUE];
- memcpy(unmapped_read_chr_str, c2b_unmapped_read_chr_name, strlen(c2b_unmapped_read_chr_name));
- unmapped_read_chr_str[strlen(c2b_unmapped_read_chr_name)] = '\t';
- unmapped_read_chr_str[strlen(c2b_unmapped_read_chr_name) + 1] = '\0';
- memcpy(rname_str, unmapped_read_chr_str, strlen(unmapped_read_chr_str));
- rname_str[strlen(unmapped_read_chr_str)] = '\0';
+ ssize_t c2b_unmapped_read_chr_name_size = strlen(c2b_unmapped_read_chr_name);
+ if (c2b_unmapped_read_chr_name_size >= c2b_globals.sam->element->rname_capacity) {
+ char *rname_resized = NULL;
+ rname_resized = realloc(c2b_globals.sam->element->rname, c2b_unmapped_read_chr_name_size + 2);
+ if (rname_resized) {
+ c2b_globals.sam->element->rname = rname_resized;
+ c2b_globals.sam->element->rname_capacity = c2b_unmapped_read_chr_name_size + 2;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize RNAME string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->rname, c2b_unmapped_read_chr_name, c2b_unmapped_read_chr_name_size);
+ c2b_globals.sam->element->rname[c2b_unmapped_read_chr_name_size] = '\t';
+ c2b_globals.sam->element->rname[c2b_unmapped_read_chr_name_size + 1] = '\0';
}
/* POS */
ssize_t pos_size = sam_field_offsets[3] - sam_field_offsets[2];
- char pos_src_str[C2B_MAX_FIELD_LENGTH_VALUE];
+ char pos_src_str[C2B_MAX_FIELD_LENGTH_VALUE] = {0};
memcpy(pos_src_str, src + sam_field_offsets[2] + 1, pos_size - 1);
pos_src_str[pos_size - 1] = '\0';
uint64_t pos_val = strtoull(pos_src_str, NULL, 10);
uint64_t start_val = pos_val - 1; /* remember, start = POS - 1 */
+ c2b_globals.sam->element->start = start_val;
+ c2b_globals.sam->element->stop = start_val; /* this will be adjusted after CIGAR string is parsed */
/* QNAME */
- char qname_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t qname_size = sam_field_offsets[0];
- memcpy(qname_str, src, qname_size);
- qname_str[qname_size] = '\0';
+ if (qname_size >= c2b_globals.sam->element->qname_capacity) {
+ char *qname_resized = NULL;
+ qname_resized = realloc(c2b_globals.sam->element->qname, qname_size + 1);
+ if (qname_resized) {
+ c2b_globals.sam->element->qname = qname_resized;
+ c2b_globals.sam->element->qname_capacity = qname_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize QNAME string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->qname, src, qname_size);
+ c2b_globals.sam->element->qname[qname_size] = '\0';
/* 16 & FLAG */
- int strand_val = 0x10 & flag_val;
- char strand_str[C2B_MAX_STRAND_LENGTH_VALUE];
+ int strand_val = 0x10 & c2b_globals.sam->element->flag;
+ char strand_str[C2B_MAX_STRAND_LENGTH_VALUE] = {0};
sprintf(strand_str, "%c", (strand_val == 0x10) ? '-' : '+');
-
+ ssize_t strand_size = strlen(strand_str);
+ if (strand_size >= c2b_globals.sam->element->strand_capacity) {
+ char *strand_resized = NULL;
+ strand_resized = realloc(c2b_globals.sam->element->strand, strand_size + 1);
+ if (strand_resized) {
+ c2b_globals.sam->element->strand = strand_resized;
+ c2b_globals.sam->element->strand_capacity = strand_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize STRAND string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->strand, strand_str, strand_size);
+ c2b_globals.sam->element->strand[strand_size] = '\0';
+
/* MAPQ */
- char mapq_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t mapq_size = sam_field_offsets[4] - sam_field_offsets[3] - 1;
- memcpy(mapq_str, src + sam_field_offsets[3] + 1, mapq_size);
- mapq_str[mapq_size] = '\0';
-
+ if (mapq_size >= c2b_globals.sam->element->mapq_capacity) {
+ char *mapq_resized = NULL;
+ mapq_resized = realloc(c2b_globals.sam->element->mapq, mapq_size + 1);
+ if (mapq_resized) {
+ c2b_globals.sam->element->mapq = mapq_resized;
+ c2b_globals.sam->element->mapq_capacity = mapq_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize MAPQ string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->mapq, src + sam_field_offsets[3] + 1, mapq_size);
+ c2b_globals.sam->element->mapq[mapq_size] = '\0';
+
/* RNEXT */
- char rnext_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t rnext_size = sam_field_offsets[6] - sam_field_offsets[5] - 1;
- memcpy(rnext_str, src + sam_field_offsets[5] + 1, rnext_size);
- rnext_str[rnext_size] = '\0';
+ if (rnext_size >= c2b_globals.sam->element->rnext_capacity) {
+ char *rnext_resized = NULL;
+ rnext_resized = realloc(c2b_globals.sam->element->rnext, rnext_size + 1);
+ if (rnext_resized) {
+ c2b_globals.sam->element->rnext = rnext_resized;
+ c2b_globals.sam->element->rnext_capacity = rnext_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize RNEXT string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->rnext, src + sam_field_offsets[5] + 1, rnext_size);
+ c2b_globals.sam->element->rnext[rnext_size] = '\0';
/* PNEXT */
- char pnext_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t pnext_size = sam_field_offsets[7] - sam_field_offsets[6] - 1;
- memcpy(pnext_str, src + sam_field_offsets[6] + 1, pnext_size);
- pnext_str[pnext_size] = '\0';
+ if (pnext_size >= c2b_globals.sam->element->pnext_capacity) {
+ char *pnext_resized = NULL;
+ pnext_resized = realloc(c2b_globals.sam->element->pnext, pnext_size + 1);
+ if (pnext_resized) {
+ c2b_globals.sam->element->pnext = pnext_resized;
+ c2b_globals.sam->element->pnext_capacity = pnext_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize PNEXT string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->pnext, src + sam_field_offsets[6] + 1, pnext_size);
+ c2b_globals.sam->element->pnext[pnext_size] = '\0';
/* TLEN */
- char tlen_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t tlen_size = sam_field_offsets[8] - sam_field_offsets[7] - 1;
- memcpy(tlen_str, src + sam_field_offsets[7] + 1, tlen_size);
- tlen_str[tlen_size] = '\0';
+ if (tlen_size >= c2b_globals.sam->element->tlen_capacity) {
+ char *tlen_resized = NULL;
+ tlen_resized = realloc(c2b_globals.sam->element->tlen, tlen_size + 1);
+ if (tlen_resized) {
+ c2b_globals.sam->element->tlen = tlen_resized;
+ c2b_globals.sam->element->tlen_capacity = tlen_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize TLEN string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->tlen, src + sam_field_offsets[7] + 1, tlen_size);
+ c2b_globals.sam->element->tlen[tlen_size] = '\0';
/* SEQ */
- char seq_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t seq_size = sam_field_offsets[9] - sam_field_offsets[8] - 1;
- memcpy(seq_str, src + sam_field_offsets[8] + 1, seq_size);
- seq_str[seq_size] = '\0';
+ if (seq_size >= c2b_globals.sam->element->seq_capacity) {
+ char *seq_resized = NULL;
+ seq_resized = realloc(c2b_globals.sam->element->seq, seq_size + 1);
+ if (seq_resized) {
+ c2b_globals.sam->element->seq = seq_resized;
+ c2b_globals.sam->element->seq_capacity = seq_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize SEQ string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->seq, src + sam_field_offsets[8] + 1, seq_size);
+ c2b_globals.sam->element->seq[seq_size] = '\0';
/* QUAL */
- char qual_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t qual_size = sam_field_offsets[10] - sam_field_offsets[9] - 1;
- memcpy(qual_str, src + sam_field_offsets[9] + 1, qual_size);
- qual_str[qual_size] = '\0';
+ if (qual_size >= c2b_globals.sam->element->qual_capacity) {
+ char *qual_resized = NULL;
+ qual_resized = realloc(c2b_globals.sam->element->qual, qual_size + 1);
+ if (qual_resized) {
+ c2b_globals.sam->element->qual = qual_resized;
+ c2b_globals.sam->element->qual_capacity = qual_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize QUAL string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->qual, src + sam_field_offsets[9] + 1, qual_size);
+ c2b_globals.sam->element->qual[qual_size] = '\0';
/* Optional fields */
- char opt_str[C2B_MAX_FIELD_LENGTH_VALUE];
+ c2b_globals.sam->element->opt[0] = '\0';
+ c2b_globals.sam->element->opt_length = 0;
if (sam_field_offsets[11] != -1) {
- opt_str[0] = '\0';
for (int field_idx = 11; field_idx <= sam_field_idx; field_idx++) {
ssize_t opt_size = sam_field_offsets[field_idx] - sam_field_offsets[field_idx - 1] - (field_idx == sam_field_idx ? 1 : 0);
- memcpy(opt_str + strlen(opt_str), src + sam_field_offsets[field_idx - 1] + 1, opt_size);
- opt_str[strlen(opt_str) + opt_size] = '\0';
+ if ((c2b_globals.sam->element->opt_length + opt_size) >= c2b_globals.sam->element->opt_capacity) {
+ char *opt_resized = NULL;
+ opt_resized = realloc(c2b_globals.sam->element->opt, c2b_globals.sam->element->opt_length + opt_size + 1);
+ if (opt_resized) {
+ c2b_globals.sam->element->opt = opt_resized;
+ c2b_globals.sam->element->opt_capacity = c2b_globals.sam->element->opt_length + opt_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize OPT string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->opt + c2b_globals.sam->element->opt_length, src + sam_field_offsets[field_idx - 1] + 1, opt_size);
+ c2b_globals.sam->element->opt[c2b_globals.sam->element->opt_length + opt_size] = '\0';
+ c2b_globals.sam->element->opt_length += opt_size;
}
}
@@ -2503,22 +3352,6 @@ c2b_line_convert_sam_to_bed_unsorted_without_split_operation(char *dest, ssize_t
Loop through operations and process a line of input based on each operation and its associated value
*/
- c2b_sam_t sam;
- sam.rname = rname_str;
- sam.start = start_val;
- sam.stop = start_val;
- sam.qname = qname_str;
- sam.flag = flag_val;
- sam.strand = strand_str;
- sam.mapq = mapq_str;
- sam.cigar = cigar_str;
- sam.rnext = rnext_str;
- sam.pnext = pnext_str;
- sam.tlen = tlen_str;
- sam.seq = seq_str;
- sam.qual = qual_str;
- sam.opt = opt_str;
-
for (op_idx = 0; op_idx < c2b_globals.sam->cigar->length; ++op_idx) {
char current_op = c2b_globals.sam->cigar->ops[op_idx].operation;
unsigned int bases = c2b_globals.sam->cigar->ops[op_idx].bases;
@@ -2529,7 +3362,7 @@ c2b_line_convert_sam_to_bed_unsorted_without_split_operation(char *dest, ssize_t
case 'D':
case '=':
case 'X':
- sam.stop += bases;
+ c2b_globals.sam->element->stop += bases;
case 'H':
case 'I':
case 'P':
@@ -2540,7 +3373,7 @@ c2b_line_convert_sam_to_bed_unsorted_without_split_operation(char *dest, ssize_t
}
}
- c2b_line_convert_sam_to_bed(sam, dest, dest_size);
+ c2b_line_convert_sam_ptr_to_bed(c2b_globals.sam->element, dest, dest_size, kFalse);
}
static void
@@ -2566,10 +3399,10 @@ c2b_line_convert_sam_to_bed_unsorted_with_split_operation(char *dest, ssize_t *d
}
else {
/* copy header line to destination stream buffer */
- char src_header_line_str[C2B_MAX_LINE_LENGTH_VALUE];
- char dest_header_line_str[C2B_MAX_LINE_LENGTH_VALUE];
+ char src_header_line_str[C2B_MAX_LINE_LENGTH_VALUE] = {0};
+ char dest_header_line_str[C2B_MAX_LINE_LENGTH_VALUE] = {0};
memcpy(src_header_line_str, src, src_size);
- src_header_line_str[src_size] = '\0';
+ //src_header_line_str[src_size] = '\0';
sprintf(dest_header_line_str, "%s\t%u\t%u\t%s\n", c2b_header_chr_name, c2b_globals.header_line_idx, (c2b_globals.header_line_idx + 1), src_header_line_str);
memcpy(dest + *dest_size, dest_header_line_str, strlen(dest_header_line_str));
*dest_size += strlen(dest_header_line_str);
@@ -2606,10 +3439,21 @@ c2b_line_convert_sam_to_bed_unsorted_with_split_operation(char *dest, ssize_t *d
*/
ssize_t cigar_size = sam_field_offsets[5] - sam_field_offsets[4];
- char cigar_str[C2B_MAX_FIELD_LENGTH_VALUE];
- memcpy(cigar_str, src + sam_field_offsets[4] + 1, cigar_size - 1);
- cigar_str[cigar_size - 1] = '\0';
- c2b_sam_cigar_str_to_ops(cigar_str);
+ if (cigar_size >= c2b_globals.sam->element->cigar_capacity) {
+ char *cigar_resized = NULL;
+ cigar_resized = realloc(c2b_globals.sam->element->cigar, cigar_size + 1);
+ if (cigar_resized) {
+ c2b_globals.sam->element->cigar = cigar_resized;
+ c2b_globals.sam->element->cigar_capacity = cigar_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize CIGAR string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->cigar, src + sam_field_offsets[4] + 1, cigar_size - 1);
+ c2b_globals.sam->element->cigar[cigar_size - 1] = '\0';
+ c2b_sam_cigar_str_to_ops(c2b_globals.sam->element->cigar);
#ifdef DEBUG
c2b_sam_debug_cigar_ops(c2b_globals.sam->cigar);
#endif
@@ -2624,97 +3468,224 @@ c2b_line_convert_sam_to_bed_unsorted_with_split_operation(char *dest, ssize_t *d
*/
ssize_t flag_size = sam_field_offsets[1] - sam_field_offsets[0];
- char flag_src_str[C2B_MAX_FIELD_LENGTH_VALUE];
+ char flag_src_str[C2B_MAX_FIELD_LENGTH_VALUE] = {0};
memcpy(flag_src_str, src + sam_field_offsets[0] + 1, flag_size);
flag_src_str[flag_size] = '\0';
- int flag_val = (int) strtol(flag_src_str, NULL, 10);
- boolean is_mapped = (boolean) !(4 & flag_val);
+ c2b_globals.sam->element->flag = (int) strtol(flag_src_str, NULL, 10);
+ boolean is_mapped = (boolean) !(4 & c2b_globals.sam->element->flag);
if ((!is_mapped) && (!c2b_globals.all_reads_flag))
- return;
+ return;
/*
Secondly, we need to retrieve RNAME, POS, QNAME parameters
*/
/* RNAME */
- char rname_str[C2B_MAX_FIELD_LENGTH_VALUE];
if (is_mapped) {
ssize_t rname_size = sam_field_offsets[2] - sam_field_offsets[1] - 1;
- memcpy(rname_str, src + sam_field_offsets[1] + 1, rname_size);
- rname_str[rname_size] = '\0';
+ if (rname_size >= c2b_globals.sam->element->rname_capacity) {
+ char *rname_resized = NULL;
+ rname_resized = realloc(c2b_globals.sam->element->rname, rname_size + 1);
+ if (rname_resized) {
+ c2b_globals.sam->element->rname = rname_resized;
+ c2b_globals.sam->element->rname_capacity = rname_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize RNAME string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->rname, src + sam_field_offsets[1] + 1, rname_size);
+ c2b_globals.sam->element->rname[rname_size] = '\0';
}
else {
- char unmapped_read_chr_str[C2B_MAX_FIELD_LENGTH_VALUE];
- memcpy(unmapped_read_chr_str, c2b_unmapped_read_chr_name, strlen(c2b_unmapped_read_chr_name));
- unmapped_read_chr_str[strlen(c2b_unmapped_read_chr_name)] = '\t';
- unmapped_read_chr_str[strlen(c2b_unmapped_read_chr_name) + 1] = '\0';
- memcpy(rname_str, unmapped_read_chr_str, strlen(unmapped_read_chr_str));
- rname_str[strlen(unmapped_read_chr_str)] = '\0';
+ ssize_t c2b_unmapped_read_chr_name_size = strlen(c2b_unmapped_read_chr_name);
+ if (c2b_unmapped_read_chr_name_size >= c2b_globals.sam->element->rname_capacity) {
+ char *rname_resized = NULL;
+ rname_resized = realloc(c2b_globals.sam->element->rname, c2b_unmapped_read_chr_name_size + 2);
+ if (rname_resized) {
+ c2b_globals.sam->element->rname = rname_resized;
+ c2b_globals.sam->element->rname_capacity = c2b_unmapped_read_chr_name_size + 2;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize RNAME string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->rname, c2b_unmapped_read_chr_name, c2b_unmapped_read_chr_name_size);
+ c2b_globals.sam->element->rname[c2b_unmapped_read_chr_name_size] = '\t';
+ c2b_globals.sam->element->rname[c2b_unmapped_read_chr_name_size + 1] = '\0';
}
/* POS */
ssize_t pos_size = sam_field_offsets[3] - sam_field_offsets[2];
- char pos_src_str[C2B_MAX_FIELD_LENGTH_VALUE];
+ char pos_src_str[C2B_MAX_FIELD_LENGTH_VALUE] = {0};
memcpy(pos_src_str, src + sam_field_offsets[2] + 1, pos_size - 1);
pos_src_str[pos_size - 1] = '\0';
uint64_t pos_val = strtoull(pos_src_str, NULL, 10);
uint64_t start_val = pos_val - 1; /* remember, start = POS - 1 */
- uint64_t stop_val = start_val + cigar_length;
+ c2b_globals.sam->element->start = start_val;
+ c2b_globals.sam->element->stop = start_val; /* this will be adjusted after CIGAR string is parsed */
/* QNAME */
- char qname_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t qname_size = sam_field_offsets[0];
- memcpy(qname_str, src, qname_size);
- qname_str[qname_size] = '\0';
+ if (qname_size >= c2b_globals.sam->element->qname_capacity) {
+ char *qname_resized = NULL;
+ qname_resized = realloc(c2b_globals.sam->element->qname, qname_size + 1);
+ if (qname_resized) {
+ c2b_globals.sam->element->qname = qname_resized;
+ c2b_globals.sam->element->qname_capacity = qname_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize QNAME string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->qname, src, qname_size);
+ c2b_globals.sam->element->qname[qname_size] = '\0';
/* 16 & FLAG */
- int strand_val = 0x10 & flag_val;
- char strand_str[C2B_MAX_STRAND_LENGTH_VALUE];
+ int strand_val = 0x10 & c2b_globals.sam->element->flag;
+ char strand_str[C2B_MAX_STRAND_LENGTH_VALUE] = {0};
sprintf(strand_str, "%c", (strand_val == 0x10) ? '-' : '+');
+ ssize_t strand_size = strlen(strand_str);
+ if (strand_size >= c2b_globals.sam->element->strand_capacity) {
+ char *strand_resized = NULL;
+ strand_resized = realloc(c2b_globals.sam->element->strand, strand_size + 1);
+ if (strand_resized) {
+ c2b_globals.sam->element->strand = strand_resized;
+ c2b_globals.sam->element->strand_capacity = strand_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize STRAND string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->strand, strand_str, strand_size);
+ c2b_globals.sam->element->strand[strand_size] = '\0';
/* MAPQ */
- char mapq_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t mapq_size = sam_field_offsets[4] - sam_field_offsets[3] - 1;
- memcpy(mapq_str, src + sam_field_offsets[3] + 1, mapq_size);
- mapq_str[mapq_size] = '\0';
+ if (mapq_size >= c2b_globals.sam->element->mapq_capacity) {
+ char *mapq_resized = NULL;
+ mapq_resized = realloc(c2b_globals.sam->element->mapq, mapq_size + 1);
+ if (mapq_resized) {
+ c2b_globals.sam->element->mapq = mapq_resized;
+ c2b_globals.sam->element->mapq_capacity = mapq_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize MAPQ string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->mapq, src + sam_field_offsets[3] + 1, mapq_size);
+ c2b_globals.sam->element->mapq[mapq_size] = '\0';
/* RNEXT */
- char rnext_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t rnext_size = sam_field_offsets[6] - sam_field_offsets[5] - 1;
- memcpy(rnext_str, src + sam_field_offsets[5] + 1, rnext_size);
- rnext_str[rnext_size] = '\0';
+ if (rnext_size >= c2b_globals.sam->element->rnext_capacity) {
+ char *rnext_resized = NULL;
+ rnext_resized = realloc(c2b_globals.sam->element->rnext, rnext_size + 1);
+ if (rnext_resized) {
+ c2b_globals.sam->element->rnext = rnext_resized;
+ c2b_globals.sam->element->rnext_capacity = rnext_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize RNEXT string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->rnext, src + sam_field_offsets[5] + 1, rnext_size);
+ c2b_globals.sam->element->rnext[rnext_size] = '\0';
/* PNEXT */
- char pnext_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t pnext_size = sam_field_offsets[7] - sam_field_offsets[6] - 1;
- memcpy(pnext_str, src + sam_field_offsets[6] + 1, pnext_size);
- pnext_str[pnext_size] = '\0';
+ if (pnext_size >= c2b_globals.sam->element->pnext_capacity) {
+ char *pnext_resized = NULL;
+ pnext_resized = realloc(c2b_globals.sam->element->pnext, pnext_size + 1);
+ if (pnext_resized) {
+ c2b_globals.sam->element->pnext = pnext_resized;
+ c2b_globals.sam->element->pnext_capacity = pnext_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize PNEXT string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->pnext, src + sam_field_offsets[6] + 1, pnext_size);
+ c2b_globals.sam->element->pnext[pnext_size] = '\0';
/* TLEN */
- char tlen_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t tlen_size = sam_field_offsets[8] - sam_field_offsets[7] - 1;
- memcpy(tlen_str, src + sam_field_offsets[7] + 1, tlen_size);
- tlen_str[tlen_size] = '\0';
+ if (tlen_size >= c2b_globals.sam->element->tlen_capacity) {
+ char *tlen_resized = NULL;
+ tlen_resized = realloc(c2b_globals.sam->element->tlen, tlen_size + 1);
+ if (tlen_resized) {
+ c2b_globals.sam->element->tlen = tlen_resized;
+ c2b_globals.sam->element->tlen_capacity = tlen_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize TLEN string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->tlen, src + sam_field_offsets[7] + 1, tlen_size);
+ c2b_globals.sam->element->tlen[tlen_size] = '\0';
/* SEQ */
- char seq_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t seq_size = sam_field_offsets[9] - sam_field_offsets[8] - 1;
- memcpy(seq_str, src + sam_field_offsets[8] + 1, seq_size);
- seq_str[seq_size] = '\0';
+ if (seq_size >= c2b_globals.sam->element->seq_capacity) {
+ char *seq_resized = NULL;
+ seq_resized = realloc(c2b_globals.sam->element->seq, seq_size + 1);
+ if (seq_resized) {
+ c2b_globals.sam->element->seq = seq_resized;
+ c2b_globals.sam->element->seq_capacity = seq_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize SEQ string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->seq, src + sam_field_offsets[8] + 1, seq_size);
+ c2b_globals.sam->element->seq[seq_size] = '\0';
/* QUAL */
- char qual_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t qual_size = sam_field_offsets[10] - sam_field_offsets[9] - 1;
- memcpy(qual_str, src + sam_field_offsets[9] + 1, qual_size);
- qual_str[qual_size] = '\0';
+ if (qual_size >= c2b_globals.sam->element->qual_capacity) {
+ char *qual_resized = NULL;
+ qual_resized = realloc(c2b_globals.sam->element->qual, qual_size + 1);
+ if (qual_resized) {
+ c2b_globals.sam->element->qual = qual_resized;
+ c2b_globals.sam->element->qual_capacity = qual_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize QUAL string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->qual, src + sam_field_offsets[9] + 1, qual_size);
+ c2b_globals.sam->element->qual[qual_size] = '\0';
- /* Optional fields */
- char opt_str[C2B_MAX_FIELD_LENGTH_VALUE];
+ c2b_globals.sam->element->opt[0] = '\0';
+ c2b_globals.sam->element->opt_length = 0;
if (sam_field_offsets[11] != -1) {
for (int field_idx = 11; field_idx <= sam_field_idx; field_idx++) {
ssize_t opt_size = sam_field_offsets[field_idx] - sam_field_offsets[field_idx - 1] - (field_idx == sam_field_idx ? 1 : 0);
- memcpy(opt_str + strlen(opt_str), src + sam_field_offsets[field_idx - 1] + 1, opt_size);
- opt_str[strlen(opt_str) + opt_size] = '\0';
+ if ((c2b_globals.sam->element->opt_length + opt_size) >= c2b_globals.sam->element->opt_capacity) {
+ char *opt_resized = NULL;
+ opt_resized = realloc(c2b_globals.sam->element->opt, c2b_globals.sam->element->opt_length + opt_size + 1);
+ if (opt_resized) {
+ c2b_globals.sam->element->opt = opt_resized;
+ c2b_globals.sam->element->opt_capacity = c2b_globals.sam->element->opt_length + opt_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize OPT string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.sam->element->opt + c2b_globals.sam->element->opt_length, src + sam_field_offsets[field_idx - 1] + 1, opt_size);
+ c2b_globals.sam->element->opt[c2b_globals.sam->element->opt_length + opt_size] = '\0';
+ c2b_globals.sam->element->opt_length += opt_size;
}
}
@@ -2724,23 +3695,6 @@ c2b_line_convert_sam_to_bed_unsorted_with_split_operation(char *dest, ssize_t *d
ssize_t block_idx;
char previous_op = default_cigar_op_operation;
- char modified_qname_str[C2B_MAX_FIELD_LENGTH_VALUE];
-
- c2b_sam_t sam;
- sam.rname = rname_str;
- sam.start = start_val;
- sam.stop = start_val;
- sam.qname = qname_str;
- sam.flag = flag_val;
- sam.strand = strand_str;
- sam.mapq = mapq_str;
- sam.cigar = cigar_str;
- sam.rnext = rnext_str;
- sam.pnext = pnext_str;
- sam.tlen = tlen_str;
- sam.seq = seq_str;
- sam.qual = qual_str;
- sam.opt = opt_str;
for (op_idx = 0, block_idx = 1; op_idx < c2b_globals.sam->cigar->length; ++op_idx) {
char current_op = c2b_globals.sam->cigar->ops[op_idx].operation;
@@ -2748,18 +3702,32 @@ c2b_line_convert_sam_to_bed_unsorted_with_split_operation(char *dest, ssize_t *d
switch (current_op)
{
case 'M':
- sam.stop += bases;
+ c2b_globals.sam->element->stop += bases;
if ((previous_op == default_cigar_op_operation) || (previous_op == 'D') || (previous_op == 'N')) {
- sprintf(modified_qname_str, "%s/%zu", qname_str, block_idx++);
- sam.qname = modified_qname_str;
- c2b_line_convert_sam_to_bed(sam, dest, dest_size);
- sam.start = stop_val;
+ ssize_t desired_modified_qname_capacity = c2b_globals.sam->element->qname_capacity + C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_EXTENSION;
+ if (c2b_globals.sam->element->modified_qname_capacity <= desired_modified_qname_capacity) {
+ // resize modified qname capacity to fit
+ char *modified_qname_resized = NULL;
+ modified_qname_resized = realloc(c2b_globals.sam->element->modified_qname, desired_modified_qname_capacity + 1);
+ if (modified_qname_resized) {
+ c2b_globals.sam->element->modified_qname = modified_qname_resized;
+ c2b_globals.sam->element->modified_qname_capacity = desired_modified_qname_capacity + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize modified QNAME string in SAM element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ // block_idx string can be up to (C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_EXTENSION-1) characters long
+ sprintf(c2b_globals.sam->element->modified_qname, "%s/%zu", c2b_globals.sam->element->qname, block_idx++);
+ c2b_line_convert_sam_ptr_to_bed(c2b_globals.sam->element, dest, dest_size, kTrue);
+ c2b_globals.sam->element->start = c2b_globals.sam->element->stop;
}
break;
case 'N':
case 'D':
- sam.stop += bases;
- sam.start = sam.stop;
+ c2b_globals.sam->element->stop += bases;
+ c2b_globals.sam->element->start = c2b_globals.sam->element->stop;
case 'H':
case 'I':
case 'P':
@@ -2778,7 +3746,7 @@ c2b_line_convert_sam_to_bed_unsorted_with_split_operation(char *dest, ssize_t *d
*/
if (block_idx == 1) {
- c2b_line_convert_sam_to_bed(sam, dest, dest_size);
+ c2b_line_convert_sam_ptr_to_bed(c2b_globals.sam->element, dest, dest_size, kFalse);
}
}
@@ -2803,6 +3771,13 @@ c2b_sam_cigar_str_to_ops(char *s)
op_idx++;
operation_flag = kFalse;
bases_flag = kTrue;
+ /* if op_idx >= size property of CIGAR entity, we need to resize this entity */
+ if (op_idx >= c2b_globals.sam->cigar->size) {
+ c2b_cigar_t *resized_cigar = NULL;
+ c2b_sam_resize_cigar_ops(&resized_cigar, c2b_globals.sam->cigar);
+ c2b_sam_delete_cigar_ops(c2b_globals.sam->cigar);
+ c2b_globals.sam->cigar = resized_cigar;
+ }
}
curr_bases_field[bases_idx++] = curr_char;
curr_bases_field[bases_idx] = '\0';
@@ -2826,21 +3801,151 @@ c2b_sam_cigar_str_to_ops(char *s)
}
static void
+c2b_sam_init_element(c2b_sam_t **e)
+{
+ *e = malloc(sizeof(c2b_sam_t));
+ if (!*e) {
+ fprintf(stderr, "Error: Could not allocate space for SAM element pointer\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+
+ (*e)->qname = NULL, (*e)->qname = malloc(C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->qname)));
+ if (!(*e)->qname) {
+ fprintf(stderr, "Error: Could not allocate space for SAM element qname malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->qname_capacity = C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->modified_qname = NULL, (*e)->modified_qname = malloc(C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->modified_qname)));
+ if (!(*e)->modified_qname) {
+ fprintf(stderr, "Error: Could not allocate space for SAM element modified qname malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->modified_qname_capacity = C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->strand = NULL, (*e)->strand = malloc(C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->strand)));
+ if (!(*e)->strand) {
+ fprintf(stderr, "Error: Could not allocate space for SAM element strand malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->strand_capacity = C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->rname = NULL, (*e)->rname = malloc(C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->rname)));
+ if (!(*e)->rname) {
+ fprintf(stderr, "Error: Could not allocate space for SAM element rname malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->rname_capacity = C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->mapq = NULL, (*e)->mapq = malloc(C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->mapq)));
+ if (!(*e)->mapq) {
+ fprintf(stderr, "Error: Could not allocate space for SAM element mapq malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->mapq_capacity = C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->cigar = NULL, (*e)->cigar = malloc(C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->cigar)));
+ if (!(*e)->cigar) {
+ fprintf(stderr, "Error: Could not allocate space for SAM element cigar malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->cigar_capacity = C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->rnext = NULL, (*e)->rnext = malloc(C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->rnext)));
+ if (!(*e)->rnext) {
+ fprintf(stderr, "Error: Could not allocate space for SAM element rnext malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->rnext_capacity = C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->pnext = NULL, (*e)->pnext = malloc(C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->pnext)));
+ if (!(*e)->pnext) {
+ fprintf(stderr, "Error: Could not allocate space for SAM element pnext malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->pnext_capacity = C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->tlen = NULL, (*e)->tlen = malloc(C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->tlen)));
+ if (!(*e)->tlen) {
+ fprintf(stderr, "Error: Could not allocate space for SAM element tlen malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->tlen_capacity = C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->seq = NULL, (*e)->seq = malloc(C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->seq)));
+ if (!(*e)->seq) {
+ fprintf(stderr, "Error: Could not allocate space for SAM element seq malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->seq_capacity = C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->qual = NULL, (*e)->qual = malloc(C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->qual)));
+ if (!(*e)->qual) {
+ fprintf(stderr, "Error: Could not allocate space for SAM element qual malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->qual_capacity = C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->opt = NULL, (*e)->opt = malloc(C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->opt)));
+ if (!(*e)->opt) {
+ fprintf(stderr, "Error: Could not allocate space for SAM element opt malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->opt_length = 0;
+ (*e)->opt_capacity = C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->flag = 0;
+ (*e)->start = 0;
+ (*e)->stop = 0;
+}
+
+static void
+c2b_sam_delete_element(c2b_sam_t *e)
+{
+ if (e->qname) { free(e->qname), e->qname = NULL; }
+ if (e->modified_qname) { free(e->modified_qname), e->modified_qname = NULL; }
+ if (e->strand) { free(e->strand), e->strand = NULL; }
+ if (e->rname) { free(e->rname), e->rname = NULL; }
+ if (e->mapq) { free(e->mapq), e->mapq = NULL; }
+ if (e->cigar) { free(e->cigar), e->cigar = NULL; }
+ if (e->rnext) { free(e->rnext), e->rnext = NULL; }
+ if (e->pnext) { free(e->pnext), e->pnext = NULL; }
+ if (e->tlen) { free(e->tlen), e->tlen = NULL; }
+ if (e->seq) { free(e->seq), e->seq = NULL; }
+ if (e->qual) { free(e->qual), e->qual = NULL; }
+ if (e->opt) { free(e->opt), e->opt = NULL; }
+ if (e) { free(e), e = NULL; }
+}
+
+static void
c2b_sam_init_cigar_ops(c2b_cigar_t **c, const ssize_t size)
{
*c = malloc(sizeof(c2b_cigar_t));
if (!*c) {
- fprintf(stderr, "Error: Could not allocate space for CIGAR struct pointer\n");
+ fprintf(stderr, "Error: Could not allocate space for SAM CIGAR struct pointer\n");
c2b_print_usage(stderr);
exit(ENOMEM); /* Not enough space (POSIX.1) */
}
- (*c)->ops = malloc(size * sizeof(c2b_cigar_op_t));
+ (*c)->size = size;
+ (*c)->ops = malloc((*c)->size * sizeof(c2b_cigar_op_t));
if (!(*c)->ops) {
- fprintf(stderr, "Error: Could not allocate space for CIGAR struct operation pointer\n");
+ fprintf(stderr, "Error: Could not allocate space for SAM CIGAR struct malloc operation pointer\n");
c2b_print_usage(stderr);
exit(ENOMEM); /* Not enough space (POSIX.1) */
}
- (*c)->size = size;
(*c)->length = 0;
for (ssize_t idx = 0; idx < size; idx++) {
(*c)->ops[idx].bases = default_cigar_op_bases;
@@ -2848,6 +3953,36 @@ c2b_sam_init_cigar_ops(c2b_cigar_t **c, const ssize_t size)
}
}
+static void
+c2b_sam_resize_cigar_ops(c2b_cigar_t **new_c, c2b_cigar_t *old_c)
+{
+ *new_c = malloc(sizeof(c2b_cigar_t));
+ if (!*new_c) {
+ fprintf(stderr, "Error: Could not allocate space for larger SAM CIGAR struct pointer\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ /* we increment the size of the input CIGAR entity by C2B_SAM_CIGAR_OPS_VALUE_INCREMENT */
+ (*new_c)->size = old_c->size + C2B_SAM_CIGAR_OPS_VALUE_INCREMENT;
+ (*new_c)->ops = malloc((*new_c)->size * sizeof(c2b_cigar_op_t));
+ if (!(*new_c)->ops) {
+ fprintf(stderr, "Error: Could not allocate space for larger SAM CIGAR struct malloc operation pointer\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*new_c)->length = old_c->length;
+ /* copy operations from input CIGAR entity to new entity */
+ for (ssize_t idx = 0; idx < old_c->size; idx++) {
+ (*new_c)->ops[idx].bases = old_c->ops[idx].bases;
+ (*new_c)->ops[idx].operation = old_c->ops[idx].operation;
+ }
+ /* set default base and operation values for new entity */
+ for (ssize_t idx = old_c->size; idx < (*new_c)->size; idx++) {
+ (*new_c)->ops[idx].bases = default_cigar_op_bases;
+ (*new_c)->ops[idx].operation = default_cigar_op_operation;
+ }
+}
+
/*
specifying special attribute for c2b_sam_debug_cigar_ops() to avoid: "warning: unused
function 'c2b_sam_debug_cigar_ops' [-Wunused-function]" message during non-debug compilation
@@ -2875,18 +4010,19 @@ c2b_sam_delete_cigar_ops(c2b_cigar_t *c)
if (c->ops) {
free(c->ops), c->ops = NULL;
}
- c->length = 0, c->size = 0;
+ c->length = 0;
+ c->size = 0;
free(c), c = NULL;
}
}
static inline void
-c2b_line_convert_sam_to_bed(c2b_sam_t s, char *dest_line, ssize_t *dest_size)
+c2b_line_convert_sam_ptr_to_bed(c2b_sam_t *s, char *dest_line, ssize_t *dest_size, boolean print_modified_qname)
{
/*
For SAM-formatted data, we use the mapping provided by BEDOPS convention described at:
- http://bedops.readthedocs.org/en/latest/content/reference/file-management/conversion/sam2bed.html
+ http://bedops.readthedocs.io/en/latest/content/reference/file-management/conversion/sam2bed.html
SAM field BED column index BED field
-------------------------------------------------------------------------
@@ -2918,7 +4054,7 @@ c2b_line_convert_sam_to_bed(c2b_sam_t s, char *dest_line, ssize_t *dest_size)
Alignment fields 14+ -
*/
- if (strlen(s.opt)) {
+ if (s->opt_length > 0) {
*dest_size += sprintf(dest_line + *dest_size,
"%s\t" \
"%" PRIu64 "\t" \
@@ -2934,21 +4070,20 @@ c2b_line_convert_sam_to_bed(c2b_sam_t s, char *dest_line, ssize_t *dest_size)
"%s\t" \
"%s\t" \
"%s\n",
- s.rname,
- s.start,
- s.stop,
- s.qname,
- s.mapq,
- s.strand,
- s.flag,
- s.cigar,
- s.rnext,
- s.pnext,
- s.tlen,
- s.seq,
- s.qual,
- s.opt);
- memset(s.opt, 0, strlen(s.opt));
+ s->rname,
+ s->start,
+ s->stop,
+ (print_modified_qname ? s->modified_qname : s->qname),
+ s->mapq,
+ s->strand,
+ s->flag,
+ s->cigar,
+ s->rnext,
+ s->pnext,
+ s->tlen,
+ s->seq,
+ s->qual,
+ s->opt);
}
else {
*dest_size += sprintf(dest_line + *dest_size,
@@ -2965,19 +4100,19 @@ c2b_line_convert_sam_to_bed(c2b_sam_t s, char *dest_line, ssize_t *dest_size)
"%s\t" \
"%s\t" \
"%s\n",
- s.rname,
- s.start,
- s.stop,
- s.qname,
- s.mapq,
- s.strand,
- s.flag,
- s.cigar,
- s.rnext,
- s.pnext,
- s.tlen,
- s.seq,
- s.qual);
+ s->rname,
+ s->start,
+ s->stop,
+ (print_modified_qname ? s->modified_qname : s->qname),
+ s->mapq,
+ s->strand,
+ s->flag,
+ s->cigar,
+ s->rnext,
+ s->pnext,
+ s->tlen,
+ s->seq,
+ s->qual);
}
}
@@ -3004,205 +4139,246 @@ c2b_line_convert_vcf_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src,
/*
If number of fields in not in bounds, we may need to exit early
*/
-
- char *src_header_line_str = NULL;
- src_header_line_str = malloc(C2B_MAX_LONGER_LINE_LENGTH_VALUE);
- if (!src_header_line_str) {
- fprintf(stderr, "Error: Could not allocate space for VCF source header line string\n");
- exit(ENOMEM); /* Not enough space (POSIX.1) */
- }
-
- char *dest_header_line_str = NULL;
- dest_header_line_str = malloc(C2B_MAX_LONGER_LINE_LENGTH_VALUE);
- if (!dest_header_line_str) {
- fprintf(stderr, "Error: Could not allocate space for VCF destination header line string\n");
- exit(ENOMEM); /* Not enough space (POSIX.1) */
- }
- if ((vcf_field_idx + 1) < c2b_vcf_field_min) {
- /* Legal header cases: line starts with "##" or "#" */
- if ((vcf_field_idx == 0) && (src[0] == c2b_vcf_header_prefix)) {
- if (c2b_globals.keep_header_flag) {
- /* copy header line to destination stream buffer */
- memcpy(src_header_line_str, src, src_size);
- src_header_line_str[src_size] = '\0';
- sprintf(dest_header_line_str, "%s\t%u\t%u\t%s\n", c2b_header_chr_name, c2b_globals.header_line_idx, (c2b_globals.header_line_idx + 1), src_header_line_str);
- fprintf(stderr, "%s", dest_header_line_str);
- memcpy(dest + *dest_size, dest_header_line_str, strlen(dest_header_line_str));
- *dest_size += strlen(dest_header_line_str);
- c2b_globals.header_line_idx++;
- free(src_header_line_str), src_header_line_str = NULL;
- free(dest_header_line_str), dest_header_line_str = NULL;
- return;
- }
- else {
- return;
- }
+ /* 0 - CHROM */
+ ssize_t chrom_size = vcf_field_offsets[0];
+ if (chrom_size >= c2b_globals.vcf->element->chrom_capacity) {
+ char *chrom_resized = NULL;
+ chrom_resized = realloc(c2b_globals.vcf->element->chrom, chrom_size + 1);
+ if (chrom_resized) {
+ c2b_globals.vcf->element->chrom = chrom_resized;
+ c2b_globals.vcf->element->chrom_capacity = chrom_size + 1;
}
- else if (vcf_field_idx == 0) {
- fprintf(stderr, "Error: Invalid field count (%d) -- input file may not match input format\n", vcf_field_idx);
- c2b_print_usage(stderr);
- exit(EINVAL); /* Invalid argument (POSIX.1) */
+ else {
+ fprintf(stderr, "Error: Could not resize CHROM string in VCF element struct\n");
+ exit(ENOMEM);
}
}
+ memcpy(c2b_globals.vcf->element->chrom, src, chrom_size);
+ c2b_globals.vcf->element->chrom[chrom_size] = '\0';
- /* 0 - CHROM */
- char chrom_str[C2B_MAX_FIELD_LENGTH_VALUE];
- ssize_t chrom_size = vcf_field_offsets[0];
- memcpy(chrom_str, src, chrom_size);
- chrom_str[chrom_size] = '\0';
+ if ((c2b_globals.vcf->element->chrom[0] == c2b_vcf_header_prefix) && (c2b_globals.keep_header_flag)) {
+ char *src_header_line_str = NULL;
+ src_header_line_str = malloc(C2B_MAX_LONGER_LINE_LENGTH_VALUE);
+ if (!src_header_line_str) {
+ fprintf(stderr, "Error: Could not allocate space for VCF source header line string\n");
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
- if ((chrom_str[0] == c2b_vcf_header_prefix) && (c2b_globals.keep_header_flag)) {
+ char *dest_header_line_str = NULL;
+ dest_header_line_str = malloc(C2B_MAX_LONGER_LINE_LENGTH_VALUE);
+ if (!dest_header_line_str) {
+ fprintf(stderr, "Error: Could not allocate space for VCF destination header line string\n");
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
memcpy(src_header_line_str, src, src_size);
src_header_line_str[src_size] = '\0';
sprintf(dest_header_line_str, "%s\t%u\t%u\t%s\n", c2b_header_chr_name, c2b_globals.header_line_idx, (c2b_globals.header_line_idx + 1), src_header_line_str);
memcpy(dest + *dest_size, dest_header_line_str, strlen(dest_header_line_str));
*dest_size += strlen(dest_header_line_str);
c2b_globals.header_line_idx++;
+ free(src_header_line_str);
+ free(dest_header_line_str);
return;
}
- else if (chrom_str[0] == c2b_vcf_header_prefix) {
+ else if (c2b_globals.vcf->element->chrom[0] == c2b_vcf_header_prefix) {
return;
}
/* 1 - POS */
char pos_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t pos_size = vcf_field_offsets[1] - vcf_field_offsets[0] - 1;
+ if (pos_size >= C2B_MAX_FIELD_LENGTH_VALUE) {
+ fprintf(stderr, "Error: Intermediate POS string too long to store in stack variable\n");
+ exit(ENOMEM);
+ }
memcpy(pos_str, src + vcf_field_offsets[0] + 1, pos_size);
pos_str[pos_size] = '\0';
- uint64_t pos_val = strtoull(pos_str, NULL, 10);
- uint64_t start_val = pos_val - 1;
- uint64_t end_val = pos_val; /* note that this value may change below, depending on options */
+ c2b_globals.vcf->element->pos = strtoull(pos_str, NULL, 10);
+ c2b_globals.vcf->element->start = c2b_globals.vcf->element->pos - 1;
+ c2b_globals.vcf->element->end = c2b_globals.vcf->element->pos; /* note that this value may change below, depending on options */
/* 2 - ID */
- char id_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t id_size = vcf_field_offsets[2] - vcf_field_offsets[1] - 1;
- memcpy(id_str, src + vcf_field_offsets[1] + 1, id_size);
- id_str[id_size] = '\0';
+ if (id_size >= c2b_globals.vcf->element->id_capacity) {
+ char *id_resized = NULL;
+ id_resized = realloc(c2b_globals.vcf->element->id, id_size + 1);
+ if (id_resized) {
+ c2b_globals.vcf->element->id = id_resized;
+ c2b_globals.vcf->element->id_capacity = id_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize ID string in VCF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.vcf->element->id, src + vcf_field_offsets[1] + 1, id_size);
+ c2b_globals.vcf->element->id[id_size] = '\0';
/* 3 - REF */
- char ref_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t ref_size = vcf_field_offsets[3] - vcf_field_offsets[2] - 1;
- memcpy(ref_str, src + vcf_field_offsets[2] + 1, ref_size);
- ref_str[ref_size] = '\0';
+ if (ref_size >= c2b_globals.vcf->element->ref_capacity) {
+ char *ref_resized = NULL;
+ ref_resized = realloc(c2b_globals.vcf->element->ref, ref_size + 1);
+ if (ref_resized) {
+ c2b_globals.vcf->element->ref = ref_resized;
+ c2b_globals.vcf->element->ref_capacity = ref_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize REF string in VCF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.vcf->element->ref, src + vcf_field_offsets[2] + 1, ref_size);
+ c2b_globals.vcf->element->ref[ref_size] = '\0';
/* 4 - ALT */
- char alt_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t alt_size = vcf_field_offsets[4] - vcf_field_offsets[3] - 1;
- memcpy(alt_str, src + vcf_field_offsets[3] + 1, alt_size);
- alt_str[alt_size] = '\0';
+ if (alt_size >= c2b_globals.vcf->element->alt_capacity) {
+ char *alt_resized = NULL;
+ alt_resized = realloc(c2b_globals.vcf->element->alt, alt_size + 1);
+ if (alt_resized) {
+ c2b_globals.vcf->element->alt = alt_resized;
+ c2b_globals.vcf->element->alt_capacity = alt_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize ALT string in VCF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.vcf->element->alt, src + vcf_field_offsets[3] + 1, alt_size);
+ c2b_globals.vcf->element->alt[alt_size] = '\0';
/* 5 - QUAL */
- char qual_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t qual_size = vcf_field_offsets[5] - vcf_field_offsets[4] - 1;
- memcpy(qual_str, src + vcf_field_offsets[4] + 1, qual_size);
- qual_str[qual_size] = '\0';
+ if (qual_size >= c2b_globals.vcf->element->qual_capacity) {
+ char *qual_resized = NULL;
+ qual_resized = realloc(c2b_globals.vcf->element->qual, qual_size + 1);
+ if (qual_resized) {
+ c2b_globals.vcf->element->qual = qual_resized;
+ c2b_globals.vcf->element->qual_capacity = qual_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize QUAL string in VCF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.vcf->element->qual, src + vcf_field_offsets[4] + 1, qual_size);
+ c2b_globals.vcf->element->qual[qual_size] = '\0';
/* 6 - FILTER */
- char filter_str[C2B_MAX_FIELD_LENGTH_VALUE];
ssize_t filter_size = vcf_field_offsets[6] - vcf_field_offsets[5] - 1;
- memcpy(filter_str, src + vcf_field_offsets[5] + 1, filter_size);
- filter_str[filter_size] = '\0';
+ if (filter_size >= c2b_globals.vcf->element->filter_capacity) {
+ char *filter_resized = NULL;
+ filter_resized = realloc(c2b_globals.vcf->element->filter, filter_size + 1);
+ if (filter_resized) {
+ c2b_globals.vcf->element->filter = filter_resized;
+ c2b_globals.vcf->element->filter_capacity = filter_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize FILTER string in VCF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.vcf->element->filter, src + vcf_field_offsets[5] + 1, filter_size);
+ c2b_globals.vcf->element->filter[filter_size] = '\0';
/* 7 - INFO */
- char *info_str = NULL;
- info_str = malloc(C2B_MAX_LONGER_LINE_LENGTH_VALUE);
- if (!info_str) {
- fprintf(stderr, "Error: Could not allocate space for VCF INFO string\n");
- exit(ENOMEM); /* Not enough space (POSIX.1) */
- }
-
ssize_t info_size = vcf_field_offsets[7] - vcf_field_offsets[6] - 1;
- memcpy(info_str, src + vcf_field_offsets[6] + 1, info_size);
- info_str[info_size] = '\0';
-
- char *format_str = NULL;
- format_str = malloc(C2B_MAX_LONGER_LINE_LENGTH_VALUE);
- if (!format_str) {
- fprintf(stderr, "Error: Could not allocate space for VCF FORMAT string\n");
- exit(ENOMEM); /* Not enough space (POSIX.1) */
+ if (info_size >= c2b_globals.vcf->element->info_capacity) {
+ char *info_resized = NULL;
+ info_resized = realloc(c2b_globals.vcf->element->info, info_size + 1);
+ if (info_resized) {
+ c2b_globals.vcf->element->info = info_resized;
+ c2b_globals.vcf->element->info_capacity = info_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize INFO string in VCF element struct\n");
+ exit(ENOMEM);
+ }
}
- format_str[0] = '\0'; /* initialize to zero-length string */
-
- char *samples_str = NULL;
- samples_str = malloc(C2B_MAX_LONGER_LINE_LENGTH_VALUE);
- if (!samples_str) {
- fprintf(stderr, "Error: Could not allocate space for VCF SAMPLE string\n");
- exit(ENOMEM); /* Not enough space (POSIX.1) */
- }
+ memcpy(c2b_globals.vcf->element->info, src + vcf_field_offsets[6] + 1, info_size);
+ c2b_globals.vcf->element->info[info_size] = '\0';
if (vcf_field_idx >= 8) {
/* 8 - FORMAT */
ssize_t format_size = vcf_field_offsets[8] - vcf_field_offsets[7] - 1;
- memcpy(format_str, src + vcf_field_offsets[7] + 1, format_size);
- format_str[format_size] = '\0';
+ if (format_size >= c2b_globals.vcf->element->format_capacity) {
+ char *format_resized = NULL;
+ format_resized = realloc(c2b_globals.vcf->element->format, format_size + 1);
+ if (format_resized) {
+ c2b_globals.vcf->element->format = format_resized;
+ c2b_globals.vcf->element->format_capacity = format_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize FORMAT string in VCF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.vcf->element->format, src + vcf_field_offsets[7] + 1, format_size);
+ c2b_globals.vcf->element->format[format_size] = '\0';
/* 9 - Samples */
ssize_t samples_size = vcf_field_offsets[vcf_field_idx] - vcf_field_offsets[8] - 1;
- memcpy(samples_str, src + vcf_field_offsets[8] + 1, samples_size);
- samples_str[samples_size] = '\0';
- }
-
- c2b_vcf_t vcf;
- vcf.chrom = chrom_str;
- vcf.pos = pos_val;
- vcf.start = start_val;
- vcf.end = end_val;
- vcf.id = id_str;
- vcf.ref = ref_str;
- vcf.alt = alt_str;
- vcf.qual = qual_str;
- vcf.filter = filter_str;
- vcf.info = info_str;
- vcf.format = format_str;
- vcf.samples = samples_str;
-
- if ((!c2b_globals.vcf->do_not_split) && (memchr(alt_str, c2b_vcf_alt_allele_delim, strlen(alt_str)))) {
+ if (samples_size >= c2b_globals.vcf->element->samples_capacity) {
+ char *samples_resized = NULL;
+ samples_resized = realloc(c2b_globals.vcf->element->samples, samples_size + 1);
+ if (samples_resized) {
+ c2b_globals.vcf->element->samples = samples_resized;
+ c2b_globals.vcf->element->samples_capacity = samples_size + 1;
+ }
+ else {
+ fprintf(stderr, "Error: Could not resize SAMPLES string in VCF element struct\n");
+ exit(ENOMEM);
+ }
+ }
+ memcpy(c2b_globals.vcf->element->samples, src + vcf_field_offsets[8] + 1, samples_size);
+ c2b_globals.vcf->element->samples[samples_size] = '\0';
+ }
+ if ((!c2b_globals.vcf->do_not_split) && (memchr(c2b_globals.vcf->element->alt, c2b_vcf_alt_allele_delim, strlen(c2b_globals.vcf->element->alt)))) {
+
/* loop through each allele */
char *alt_alleles_copy = NULL;
- alt_alleles_copy = malloc(strlen(alt_str) + 1);
+ alt_alleles_copy = malloc(strlen(c2b_globals.vcf->element->alt) + 1);
if (!alt_alleles_copy) {
- fprintf(stderr, "Error: Could not allocate space for VCF alt alleles copy\n");
+ fprintf(stderr, "Error: Could not allocate space for VCF ALT alleles copy\n");
exit(ENOMEM); /* Not enough space (POSIX.1) */
}
- memcpy(alt_alleles_copy, alt_str, strlen(alt_str) + 1);
+ memcpy(alt_alleles_copy, c2b_globals.vcf->element->alt, strlen(c2b_globals.vcf->element->alt) + 1);
const char *allele_tok;
while ((allele_tok = c2b_strsep(&alt_alleles_copy, ",")) != NULL) {
- vcf.alt = (char *) allele_tok; /* discard const */
+ memcpy(c2b_globals.vcf->element->alt, allele_tok, strlen(allele_tok));
+ c2b_globals.vcf->element->alt[strlen(allele_tok)] = '\0';
if ((c2b_globals.vcf->filter_count == 1) && (!c2b_globals.vcf->only_insertions)) {
- vcf.end = start_val + ref_size - strlen(vcf.alt) + 1;
+ c2b_globals.vcf->element->end = c2b_globals.vcf->element->start + ref_size - strlen(c2b_globals.vcf->element->alt) + 1;
}
if ( (c2b_globals.vcf->filter_count == 0) ||
- ((c2b_globals.vcf->only_snvs) && (c2b_vcf_record_is_snv(ref_str, vcf.alt))) ||
- ((c2b_globals.vcf->only_insertions) && (c2b_vcf_record_is_insertion(ref_str, vcf.alt))) ||
- ((c2b_globals.vcf->only_deletions) && (c2b_vcf_record_is_deletion(ref_str, vcf.alt))) )
+ ((c2b_globals.vcf->only_snvs) && (c2b_vcf_record_is_snv(c2b_globals.vcf->element->ref, c2b_globals.vcf->element->alt))) ||
+ ((c2b_globals.vcf->only_insertions) && (c2b_vcf_record_is_insertion(c2b_globals.vcf->element->ref, c2b_globals.vcf->element->alt))) ||
+ ((c2b_globals.vcf->only_deletions) && (c2b_vcf_record_is_deletion(c2b_globals.vcf->element->ref, c2b_globals.vcf->element->alt))) )
{
- c2b_line_convert_vcf_to_bed(vcf, dest, dest_size);
+ c2b_line_convert_vcf_ptr_to_bed(c2b_globals.vcf->element, dest, dest_size);
}
}
free(alt_alleles_copy), alt_alleles_copy = NULL;
}
else {
-
+
/* just print the one allele */
if ((c2b_globals.vcf->filter_count == 1) && (!c2b_globals.vcf->only_insertions)) {
- vcf.end = start_val + ref_size - strlen(alt_str) + 1;
+ c2b_globals.vcf->element->end = c2b_globals.vcf->element->start + ref_size - strlen(c2b_globals.vcf->element->alt) + 1;
}
if ( (c2b_globals.vcf->filter_count == 0) ||
- ((c2b_globals.vcf->only_snvs) && (c2b_vcf_record_is_snv(ref_str, alt_str))) ||
- ((c2b_globals.vcf->only_insertions) && (c2b_vcf_record_is_insertion(ref_str, alt_str))) ||
- ((c2b_globals.vcf->only_deletions) && (c2b_vcf_record_is_deletion(ref_str, alt_str))) )
+ ((c2b_globals.vcf->only_snvs) && (c2b_vcf_record_is_snv(c2b_globals.vcf->element->ref, c2b_globals.vcf->element->alt))) ||
+ ((c2b_globals.vcf->only_insertions) && (c2b_vcf_record_is_insertion(c2b_globals.vcf->element->ref, c2b_globals.vcf->element->alt))) ||
+ ((c2b_globals.vcf->only_deletions) && (c2b_vcf_record_is_deletion(c2b_globals.vcf->element->ref, c2b_globals.vcf->element->alt))) )
{
- c2b_line_convert_vcf_to_bed(vcf, dest, dest_size);
+ c2b_line_convert_vcf_ptr_to_bed(c2b_globals.vcf->element, dest, dest_size);
}
}
-
- free(info_str), info_str = NULL;
- free(format_str), format_str = NULL;
- free(samples_str), samples_str = NULL;
}
static inline boolean
@@ -3229,13 +4405,115 @@ c2b_vcf_record_is_deletion(char *ref, char *alt)
return ((!c2b_vcf_allele_is_id(alt)) && (((int) strlen(ref) - (int) strlen(alt)) > 0)) ? kTrue : kFalse;
}
+static void
+c2b_vcf_init_element(c2b_vcf_t **e)
+{
+ *e = malloc(sizeof(c2b_vcf_t));
+ if (!*e) {
+ fprintf(stderr, "Error: Could not allocate space for VCF element pointer\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+
+ (*e)->chrom = NULL, (*e)->chrom = malloc(C2B_VCF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->chrom)));
+ if (!(*e)->chrom) {
+ fprintf(stderr, "Error: Could not allocate space for VCF element chrom malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->chrom_capacity = C2B_VCF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->pos = 0;
+ (*e)->start = 0;
+ (*e)->end = 0;
+
+ (*e)->id = NULL, (*e)->id = malloc(C2B_VCF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->id)));
+ if (!(*e)->id) {
+ fprintf(stderr, "Error: Could not allocate space for VCF element id malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->id_capacity = C2B_VCF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->ref = NULL, (*e)->ref = malloc(C2B_VCF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->ref)));
+ if (!(*e)->ref) {
+ fprintf(stderr, "Error: Could not allocate space for VCF element ref malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->ref_capacity = C2B_VCF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->alt = NULL, (*e)->alt = malloc(C2B_VCF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->alt)));
+ if (!(*e)->alt) {
+ fprintf(stderr, "Error: Could not allocate space for VCF element alt malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->alt_capacity = C2B_VCF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->qual = NULL, (*e)->qual = malloc(C2B_VCF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->qual)));
+ if (!(*e)->qual) {
+ fprintf(stderr, "Error: Could not allocate space for VCF element qual malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->qual_capacity = C2B_VCF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->filter = NULL, (*e)->filter = malloc(C2B_VCF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->filter)));
+ if (!(*e)->filter) {
+ fprintf(stderr, "Error: Could not allocate space for VCF element filter malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->filter_capacity = C2B_VCF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->info = NULL, (*e)->info = malloc(C2B_VCF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->info)));
+ if (!(*e)->info) {
+ fprintf(stderr, "Error: Could not allocate space for VCF element info malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->info_capacity = C2B_VCF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->format = NULL, (*e)->format = malloc(C2B_VCF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->format)));
+ if (!(*e)->format) {
+ fprintf(stderr, "Error: Could not allocate space for VCF element format malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->format_capacity = C2B_VCF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+
+ (*e)->samples = NULL, (*e)->samples = malloc(C2B_VCF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL * sizeof(*((*e)->samples)));
+ if (!(*e)->samples) {
+ fprintf(stderr, "Error: Could not allocate space for VCF element samples malloc operation\n");
+ c2b_print_usage(stderr);
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ (*e)->samples_capacity = C2B_VCF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL;
+}
+
+static void
+c2b_vcf_delete_element(c2b_vcf_t *e)
+{
+ if (e->chrom) { free(e->chrom), e->chrom = NULL; }
+ if (e->id) { free(e->id), e->id = NULL; }
+ if (e->ref) { free(e->ref), e->ref = NULL; }
+ if (e->alt) { free(e->alt), e->alt = NULL; }
+ if (e->qual) { free(e->qual), e->qual = NULL; }
+ if (e->filter) { free(e->filter), e->filter = NULL; }
+ if (e->info) { free(e->info), e->info = NULL; }
+ if (e->format) { free(e->format), e->format = NULL; }
+ if (e->samples) { free(e->samples), e->samples = NULL; }
+ if (e) { free(e), e = NULL; }
+}
+
static inline void
-c2b_line_convert_vcf_to_bed(c2b_vcf_t v, char *dest_line, ssize_t *dest_size)
+c2b_line_convert_vcf_ptr_to_bed(c2b_vcf_t *v, char *dest_line, ssize_t *dest_size)
{
/*
For VCF v4.2-formatted data, we use the mapping provided by BEDOPS convention described at:
- http://bedops.readthedocs.org/en/latest/content/reference/file-management/conversion/vcf2bed.html
+ http://bedops.readthedocs.io/en/latest/content/reference/file-management/conversion/vcf2bed.html
VCF field BED column index BED field
-------------------------------------------------------------------------
@@ -3267,7 +4545,7 @@ c2b_line_convert_vcf_to_bed(c2b_vcf_t v, char *dest_line, ssize_t *dest_size)
...
*/
- if (strlen(v.format) > 0) {
+ if (strlen(v->format) > 0) {
*dest_size += sprintf(dest_line + *dest_size,
"%s\t" \
"%" PRIu64 "\t" \
@@ -3280,17 +4558,17 @@ c2b_line_convert_vcf_to_bed(c2b_vcf_t v, char *dest_line, ssize_t *dest_size)
"%s\t" \
"%s\t" \
"%s\n",
- v.chrom,
- v.start,
- v.end,
- v.id,
- v.qual,
- v.ref,
- v.alt,
- v.filter,
- v.info,
- v.format,
- v.samples);
+ v->chrom,
+ v->start,
+ v->end,
+ v->id,
+ v->qual,
+ v->ref,
+ v->alt,
+ v->filter,
+ v->info,
+ v->format,
+ v->samples);
}
else {
*dest_size += sprintf(dest_line + *dest_size,
@@ -3303,15 +4581,15 @@ c2b_line_convert_vcf_to_bed(c2b_vcf_t v, char *dest_line, ssize_t *dest_size)
"%s\t" \
"%s\t" \
"%s\n",
- v.chrom,
- v.start,
- v.end,
- v.id,
- v.qual,
- v.ref,
- v.alt,
- v.filter,
- v.info);
+ v->chrom,
+ v->start,
+ v->end,
+ v->id,
+ v->qual,
+ v->ref,
+ v->alt,
+ v->filter,
+ v->info);
}
}
@@ -3662,7 +4940,8 @@ c2b_read_bytes_from_stdin(void *arg)
{
c2b_pipeline_stage_t *stage = (c2b_pipeline_stage_t *) arg;
c2b_pipeset_t *pipes = stage->pipeset;
- char buffer[C2B_MAX_LINE_LENGTH_VALUE];
+ char *buffer = NULL;
+ ssize_t buffer_size = stage->buffer_size;
ssize_t bytes_read;
int exit_status = 0;
@@ -3670,14 +4949,22 @@ c2b_read_bytes_from_stdin(void *arg)
fprintf(stderr, "\t-> c2b_read_bytes_from_stdin | reading from fd (%02d) | writing to fd (%02d)\n", STDIN_FILENO, pipes->in[stage->dest][PIPE_WRITE]);
#endif
+ buffer = (char *) malloc(buffer_size);
+ if (!buffer) {
+ fprintf(stderr, "Error: Could not allocate space to c2b_read_bytes_from_stdin() buffer\n");
+ exit(ENOMEM);
+ }
+
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
- while ((bytes_read = read(STDIN_FILENO, buffer, C2B_MAX_LINE_LENGTH_VALUE)) > 0) {
+ while ((bytes_read = read(STDIN_FILENO, buffer, buffer_size)) > 0) {
write(pipes->in[stage->dest][PIPE_WRITE], buffer, bytes_read);
}
#pragma GCC diagnostic pop
close(pipes->in[stage->dest][PIPE_WRITE]);
+ free(buffer), buffer = NULL;
+
if (WIFEXITED(stage->status) || WIFSIGNALED(stage->status)) {
waitpid(stage->pid, &stage->status, WUNTRACED);
exit_status = WEXITSTATUS(stage->status);
@@ -3689,7 +4976,6 @@ c2b_read_bytes_from_stdin(void *arg)
exit_status);
}
-
pthread_exit(NULL);
}
@@ -3860,20 +5146,29 @@ c2b_write_in_bytes_to_in_process(void *arg)
{
c2b_pipeline_stage_t *stage = (c2b_pipeline_stage_t *) arg;
c2b_pipeset_t *pipes = stage->pipeset;
- char buffer[C2B_MAX_LINE_LENGTH_VALUE];
+ char *buffer = NULL;
+ ssize_t buffer_size = stage->buffer_size;
ssize_t bytes_read;
int exit_status = 0;
+ buffer = (char *) malloc(buffer_size);
+ if (!buffer) {
+ fprintf(stderr, "Error: Could not allocate space to c2b_write_in_bytes_to_in_process() buffer\n");
+ exit(ENOMEM);
+ }
+
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
/* read buffer from p->in[1] and write buffer to p->in[2] */
- while ((bytes_read = read(pipes->in[stage->src][PIPE_READ], buffer, C2B_MAX_LINE_LENGTH_VALUE)) > 0) {
+ while ((bytes_read = read(pipes->in[stage->src][PIPE_READ], buffer, buffer_size)) > 0) {
write(pipes->in[stage->dest][PIPE_WRITE], buffer, bytes_read);
}
#pragma GCC diagnostic pop
close(pipes->in[stage->dest][PIPE_WRITE]);
+ free(buffer), buffer = NULL;
+
if (WIFEXITED(stage->status) || WIFSIGNALED(stage->status)) {
waitpid(stage->pid, &stage->status, WUNTRACED);
exit_status = WEXITSTATUS(stage->status);
@@ -3893,20 +5188,29 @@ c2b_write_out_bytes_to_in_process(void *arg)
{
c2b_pipeline_stage_t *stage = (c2b_pipeline_stage_t *) arg;
c2b_pipeset_t *pipes = stage->pipeset;
- char buffer[C2B_MAX_LINE_LENGTH_VALUE];
+ char *buffer = NULL;
+ ssize_t buffer_size = stage->buffer_size;
ssize_t bytes_read;
int exit_status = 0;
+ buffer = (char *) malloc(buffer_size);
+ if (!buffer) {
+ fprintf(stderr, "Error: Could not allocate space to c2b_write_out_bytes_to_in_process() buffer\n");
+ exit(ENOMEM);
+ }
+
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
/* read buffer from p->out[1] and write buffer to p->in[2] */
- while ((bytes_read = read(pipes->out[stage->src][PIPE_READ], buffer, C2B_MAX_LINE_LENGTH_VALUE)) > 0) {
+ while ((bytes_read = read(pipes->out[stage->src][PIPE_READ], buffer, buffer_size)) > 0) {
write(pipes->in[stage->dest][PIPE_WRITE], buffer, bytes_read);
}
#pragma GCC diagnostic pop
close(pipes->in[stage->dest][PIPE_WRITE]);
+ free(buffer), buffer = NULL;
+
if (WIFEXITED(stage->status) || WIFSIGNALED(stage->status)) {
waitpid(stage->pid, &stage->status, WUNTRACED);
exit_status = WEXITSTATUS(stage->status);
@@ -3926,17 +5230,26 @@ c2b_write_in_bytes_to_stdout(void *arg)
{
c2b_pipeline_stage_t *stage = (c2b_pipeline_stage_t *) arg;
c2b_pipeset_t *pipes = stage->pipeset;
- char buffer[C2B_MAX_LINE_LENGTH_VALUE];
+ char *buffer = NULL;
+ ssize_t buffer_size = stage->buffer_size;
ssize_t bytes_read;
int exit_status = 0;
+ buffer = (char *) malloc(buffer_size);
+ if (!buffer) {
+ fprintf(stderr, "Error: Could not allocate space to c2b_write_in_bytes_to_stdout() buffer\n");
+ exit(ENOMEM);
+ }
+
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
- while ((bytes_read = read(pipes->in[stage->src][PIPE_READ], buffer, C2B_MAX_LINE_LENGTH_VALUE)) > 0) {
+ while ((bytes_read = read(pipes->in[stage->src][PIPE_READ], buffer, buffer_size)) > 0) {
write(STDOUT_FILENO, buffer, bytes_read);
}
#pragma GCC diagnostic pop
+ free(buffer), buffer = NULL;
+
if (WIFEXITED(stage->status) || WIFSIGNALED(stage->status)) {
waitpid(stage->pid, &stage->status, WUNTRACED);
exit_status = WEXITSTATUS(stage->status);
@@ -3956,17 +5269,26 @@ c2b_write_out_bytes_to_stdout(void *arg)
{
c2b_pipeline_stage_t *stage = (c2b_pipeline_stage_t *) arg;
c2b_pipeset_t *pipes = stage->pipeset;
- char buffer[C2B_MAX_LINE_LENGTH_VALUE];
+ char *buffer = NULL;
+ ssize_t buffer_size = stage->buffer_size;
ssize_t bytes_read;
int exit_status = 0;
+ buffer = (char *) malloc(buffer_size);
+ if (!buffer) {
+ fprintf(stderr, "Error: Could not allocate space to c2b_write_out_bytes_to_stdout() buffer\n");
+ exit(ENOMEM);
+ }
+
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
- while ((bytes_read = read(pipes->out[stage->src][PIPE_READ], buffer, C2B_MAX_LINE_LENGTH_VALUE)) > 0) {
+ while ((bytes_read = read(pipes->out[stage->src][PIPE_READ], buffer, buffer_size)) > 0) {
write(STDOUT_FILENO, buffer, bytes_read);
}
#pragma GCC diagnostic pop
+ free(buffer), buffer = NULL;
+
if (WIFEXITED(stage->status) || WIFSIGNALED(stage->status)) {
waitpid(stage->pid, &stage->status, WUNTRACED);
exit_status = WEXITSTATUS(stage->status);
@@ -4505,16 +5827,26 @@ c2b_init_globals()
c2b_globals.split_flag = kFalse;
c2b_globals.zero_indexed_flag = kFalse;
c2b_globals.header_line_idx = 0U;
- c2b_globals.gff = NULL, c2b_init_global_gff_state();
- c2b_globals.gtf = NULL, c2b_init_global_gtf_state();
- c2b_globals.psl = NULL, c2b_init_global_psl_state();
- c2b_globals.rmsk = NULL, c2b_init_global_rmsk_state();
- c2b_globals.sam = NULL, c2b_init_global_sam_state();
- c2b_globals.vcf = NULL, c2b_init_global_vcf_state();
- c2b_globals.wig = NULL, c2b_init_global_wig_state();
- c2b_globals.cat = NULL, c2b_init_global_cat_params();
- c2b_globals.sort = NULL, c2b_init_global_sort_params();
- c2b_globals.starch = NULL, c2b_init_global_starch_params();
+ c2b_globals.gff = NULL;
+ c2b_init_global_gff_state();
+ c2b_globals.gtf = NULL;
+ c2b_init_global_gtf_state();
+ c2b_globals.psl = NULL;
+ c2b_init_global_psl_state();
+ c2b_globals.rmsk = NULL;
+ c2b_init_global_rmsk_state();
+ c2b_globals.sam = NULL;
+ c2b_init_global_sam_state();
+ c2b_globals.vcf = NULL;
+ c2b_init_global_vcf_state();
+ c2b_globals.wig = NULL;
+ c2b_init_global_wig_state();
+ c2b_globals.cat = NULL;
+ c2b_init_global_cat_params();
+ c2b_globals.sort = NULL;
+ c2b_init_global_sort_params();
+ c2b_globals.starch = NULL;
+ c2b_init_global_starch_params();
#ifdef DEBUG
fprintf(stderr, "--- c2b_init_globals() - exit ---\n");
@@ -4564,12 +5896,7 @@ c2b_init_global_gff_state()
exit(ENOMEM); /* Not enough space (POSIX.1) */
}
- c2b_globals.gff->id = malloc(C2B_MAX_FIELD_LENGTH_VALUE);
- if (!c2b_globals.gff->id) {
- fprintf(stderr, "Error: Could not allocate space for GFF ID global\n");
- exit(ENOMEM); /* Not enough space (POSIX.1) */
- }
- memset(c2b_globals.gff->id, 0, C2B_MAX_FIELD_LENGTH_VALUE);
+ c2b_globals.gff->element = NULL, c2b_gff_init_element(&(c2b_globals.gff->element));
#ifdef DEBUG
fprintf(stderr, "--- c2b_init_global_gff_state() - exit ---\n");
@@ -4583,8 +5910,10 @@ c2b_delete_global_gff_state()
fprintf(stderr, "--- c2b_delete_global_gff_state() - enter ---\n");
#endif
- if (c2b_globals.gff->id)
- free(c2b_globals.gff->id), c2b_globals.gff->id = NULL;
+ if (c2b_globals.gff->element) {
+ c2b_gff_delete_element(c2b_globals.gff->element);
+ c2b_globals.gff->element = NULL;
+ }
free(c2b_globals.gff), c2b_globals.gff = NULL;
@@ -4606,12 +5935,7 @@ c2b_init_global_gtf_state()
exit(ENOMEM); /* Not enough space (POSIX.1) */
}
- c2b_globals.gtf->id = malloc(C2B_MAX_FIELD_LENGTH_VALUE);
- if (!c2b_globals.gtf->id) {
- fprintf(stderr, "Error: Could not allocate space for GTF ID global\n");
- exit(ENOMEM); /* Not enough space (POSIX.1) */
- }
- memset(c2b_globals.gtf->id, 0, C2B_MAX_FIELD_LENGTH_VALUE);
+ c2b_globals.gtf->element = NULL, c2b_gtf_init_element(&(c2b_globals.gtf->element));
c2b_globals.gtf->line_count = 0;
@@ -4627,8 +5951,10 @@ c2b_delete_global_gtf_state()
fprintf(stderr, "--- c2b_delete_global_gtf_state() - enter ---\n");
#endif
- if (c2b_globals.gtf->id)
- free(c2b_globals.gtf->id), c2b_globals.gtf->id = NULL;
+ if (c2b_globals.gtf->element) {
+ c2b_gtf_delete_element(c2b_globals.gtf->element);
+ c2b_globals.gtf->element = NULL;
+ }
free(c2b_globals.gtf), c2b_globals.gtf = NULL;
@@ -4713,6 +6039,7 @@ c2b_init_global_rmsk_state()
exit(ENOMEM); /* Not enough space (POSIX.1) */
}
+ c2b_globals.rmsk->element = NULL, c2b_rmsk_init_element(&(c2b_globals.rmsk->element));
c2b_globals.rmsk->line = 0U;
c2b_globals.rmsk->is_start_of_line = kTrue;
c2b_globals.rmsk->is_start_of_gap = kFalse;
@@ -4729,6 +6056,7 @@ c2b_delete_global_rmsk_state()
fprintf(stderr, "--- c2b_delete_global_rmsk_state() - enter ---\n");
#endif
+ c2b_rmsk_delete_element(c2b_globals.rmsk->element);
free(c2b_globals.rmsk), c2b_globals.rmsk = NULL;
#ifdef DEBUG
@@ -4751,7 +6079,9 @@ c2b_init_global_sam_state()
c2b_globals.sam->samtools_path = NULL;
- c2b_globals.sam->cigar = NULL, c2b_sam_init_cigar_ops(&(c2b_globals.sam->cigar), C2B_MAX_OPERATIONS_VALUE);
+ c2b_globals.sam->cigar = NULL, c2b_sam_init_cigar_ops(&(c2b_globals.sam->cigar), C2B_SAM_CIGAR_OPS_VALUE_INITIAL);
+
+ c2b_globals.sam->element = NULL, c2b_sam_init_element(&(c2b_globals.sam->element));
#ifdef DEBUG
fprintf(stderr, "--- c2b_init_global_sam_state() - exit ---\n");
@@ -4770,6 +6100,9 @@ c2b_delete_global_sam_state()
if (c2b_globals.sam->cigar)
c2b_sam_delete_cigar_ops(c2b_globals.sam->cigar);
+
+ if (c2b_globals.sam->element)
+ c2b_sam_delete_element(c2b_globals.sam->element);
free(c2b_globals.sam), c2b_globals.sam = NULL;
@@ -4796,6 +6129,7 @@ c2b_init_global_vcf_state()
c2b_globals.vcf->only_insertions = kFalse;
c2b_globals.vcf->only_deletions = kFalse;
c2b_globals.vcf->filter_count = 0U;
+ c2b_globals.vcf->element = NULL, c2b_vcf_init_element(&(c2b_globals.vcf->element));
#ifdef DEBUG
fprintf(stderr, "--- c2b_init_global_vcf_state() - exit ---\n");
@@ -4809,6 +6143,11 @@ c2b_delete_global_vcf_state()
fprintf(stderr, "--- c2b_delete_global_vcf_state() - enter ---\n");
#endif
+ if (c2b_globals.vcf->element) {
+ c2b_vcf_delete_element(c2b_globals.vcf->element);
+ c2b_globals.vcf->element = NULL;
+ }
+
free(c2b_globals.vcf), c2b_globals.vcf = NULL;
#ifdef DEBUG
diff --git a/applications/bed/conversion/src/convert2bed.h b/applications/bed/conversion/src/convert2bed.h
index ec0057a..bfd2d22 100644
--- a/applications/bed/conversion/src/convert2bed.h
+++ b/applications/bed/conversion/src/convert2bed.h
@@ -1,9 +1,9 @@
/*
convert2bed.h
-----------------------------------------------------------------------
- Copyright (C) 2014-2016 Alex Reynolds
+ Copyright (C) 2014-2017 Alex Reynolds
- wig2bed components, (C) 2011-2016 Scott Kuehn and Shane Neph
+ wig2bed components, (C) 2011-2017 Scott Kuehn and Shane Neph
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -51,7 +51,7 @@
#include <sys/param.h>
#include <sys/wait.h>
-#define C2B_VERSION "2.4.20"
+#define C2B_VERSION "2.4.26"
typedef int boolean;
extern const boolean kTrue;
@@ -66,12 +66,20 @@ const boolean kFalse = 0;
#define C2B_MAX_LINE_LENGTH_VALUE 131072
#define C2B_MAX_LONGER_LINE_LENGTH_VALUE 1048576
#define C2B_MAX_LINES_VALUE 32
-#define C2B_MAX_OPERATIONS_VALUE 1024
#define C2B_MAX_CHROMOSOME_LENGTH 32
#define C2B_MAX_PSL_BLOCKS 1024
#define C2B_MAX_PSL_BLOCK_SIZES_STRING_LENGTH 20
#define C2B_MAX_PSL_T_STARTS_STRING_LENGTH 20
#define C2B_MAX_VCF_FIELD_COUNT_VALUE 24576
+#define C2B_SAM_CIGAR_OPS_VALUE_INITIAL 1
+#define C2B_SAM_CIGAR_OPS_VALUE_INCREMENT 1000
+#define C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_INITIAL 32
+#define C2B_SAM_ELEMENT_FIELD_LENGTH_VALUE_EXTENSION 32
+#define C2B_VCF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL 32
+#define C2B_GFF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL 32
+#define C2B_GTF_ELEMENT_FIELD_LENGTH_VALUE_INITIAL 32
+#define C2B_RMSK_ELEMENT_FIELD_LENGTH_VALUE_INITIAL 32
+#define C2B_THREAD_IO_BUFFER_SIZE 5000000
extern const char *c2b_samtools;
extern const char *c2b_sort_bed;
@@ -254,19 +262,33 @@ const char default_cigar_op_operation = '-';
typedef struct sam {
char *qname;
+ ssize_t qname_capacity;
+ char *modified_qname;
+ ssize_t modified_qname_capacity;
int flag;
char *strand;
+ ssize_t strand_capacity;
char *rname;
+ ssize_t rname_capacity;
uint64_t start;
uint64_t stop;
char *mapq;
+ ssize_t mapq_capacity;
char *cigar;
+ ssize_t cigar_capacity;
char *rnext;
+ ssize_t rnext_capacity;
char *pnext;
+ ssize_t pnext_capacity;
char *tlen;
+ ssize_t tlen_capacity;
char *seq;
+ ssize_t seq_capacity;
char *qual;
+ ssize_t qual_capacity;
char *opt;
+ ssize_t opt_length;
+ ssize_t opt_capacity;
} c2b_sam_t;
/*
@@ -291,15 +313,23 @@ typedef struct sam {
typedef struct gff {
char *seqid;
+ ssize_t seqid_capacity;
char *source;
+ ssize_t source_capacity;
char *type;
+ ssize_t type_capacity;
uint64_t start;
uint64_t end;
char *score;
+ ssize_t score_capacity;
char *strand;
+ ssize_t strand_capacity;
char *phase;
+ ssize_t phase_capacity;
char *attributes;
+ ssize_t attributes_capacity;
char *id;
+ ssize_t id_capacity;
} c2b_gff_t;
/*
@@ -325,16 +355,25 @@ typedef struct gff {
typedef struct gtf {
char *seqname;
+ ssize_t seqname_capacity;
char *source;
+ ssize_t source_capacity;
char *feature;
+ ssize_t feature_capacity;
uint64_t start;
uint64_t end;
char *score;
+ ssize_t score_capacity;
char *strand;
+ ssize_t strand_capacity;
char *frame;
+ ssize_t frame_capacity;
char *attributes;
+ ssize_t attributes_capacity;
char *id;
+ ssize_t id_capacity;
char *comments;
+ ssize_t comments_capacity;
} c2b_gtf_t;
/*
@@ -433,21 +472,35 @@ typedef struct block {
typedef struct rmsk {
char *sw_score;
+ ssize_t sw_score_capacity;
char *perc_div;
+ ssize_t perc_div_capacity;
char *perc_deleted;
+ ssize_t perc_deleted_capacity;
char *perc_inserted;
+ ssize_t perc_inserted_capacity;
char *query_seq;
+ ssize_t query_seq_capacity;
uint64_t query_start;
uint64_t query_end;
char *bases_past_match;
+ ssize_t bases_past_match_capacity;
char *strand;
+ ssize_t strand_capacity;
char *repeat_name;
+ ssize_t repeat_name_capacity;
char *repeat_class;
+ ssize_t repeat_class_capacity;
char *bases_before_match_comp;
+ ssize_t bases_before_match_comp_capacity;
char *match_start;
+ ssize_t match_start_capacity;
char *match_end;
+ ssize_t match_end_capacity;
char *unique_id;
+ ssize_t unique_id_capacity;
char *higher_score_match;
+ ssize_t higher_score_match_capacity;
} c2b_rmsk_t;
/*
@@ -482,17 +535,26 @@ typedef struct rmsk {
typedef struct vcf {
char *chrom;
+ ssize_t chrom_capacity;
uint64_t pos;
uint64_t start;
uint64_t end;
char *id;
+ ssize_t id_capacity;
char *ref;
+ ssize_t ref_capacity;
char *alt;
+ ssize_t alt_capacity;
char *qual;
+ ssize_t qual_capacity;
char *filter;
+ ssize_t filter_capacity;
char *info;
+ ssize_t info_capacity;
char *format;
+ ssize_t format_capacity;
char *samples;
+ ssize_t samples_capacity;
} c2b_vcf_t;
/*
@@ -552,6 +614,7 @@ typedef struct pipeline_stage {
int status;
char *description;
pid_t pid;
+ ssize_t buffer_size;
} c2b_pipeline_stage_t;
#define PIPE4_FLAG_NONE (0U)
@@ -1208,11 +1271,11 @@ static const char *format_undefined_usage = \
" --help[-bam|-gff|-gtf|-gvf|-psl|-rmsk|-sam|-vcf|-wig] (-h <fmt>)\n";
typedef struct gff_state {
- char *id;
+ c2b_gff_t *element;
} c2b_gff_state_t;
typedef struct gtf_state {
- char *id;
+ c2b_gtf_t *element;
uint64_t line_count;
} c2b_gtf_state_t;
@@ -1222,6 +1285,7 @@ typedef struct psl_state {
} c2b_psl_state_t;
typedef struct rmsk_state {
+ c2b_rmsk_t *element;
uint64_t line;
boolean is_start_of_line;
boolean is_start_of_gap;
@@ -1230,6 +1294,7 @@ typedef struct rmsk_state {
typedef struct sam_state {
char *samtools_path;
c2b_cigar_t *cigar;
+ c2b_sam_t *element;
} c2b_sam_state_t;
typedef struct vcf_state {
@@ -1238,6 +1303,7 @@ typedef struct vcf_state {
boolean only_insertions;
boolean only_deletions;
unsigned int filter_count;
+ c2b_vcf_t *element;
} c2b_vcf_state_t;
typedef struct wig_state {
@@ -1352,29 +1418,40 @@ extern "C" {
static inline void c2b_cmd_bam_to_sam(char *cmd);
static inline void c2b_cmd_sort_bed(char *cmd);
static inline void c2b_cmd_starch_bed(char *cmd);
- static void c2b_line_convert_gff_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src, ssize_t src_size);
- static inline void c2b_line_convert_gff_to_bed(c2b_gff_t g, char *dest_line, ssize_t *dest_size);
+ static void c2b_gtf_init_element(c2b_gtf_t **e);
+ static void c2b_gtf_delete_element(c2b_gtf_t *e);
static void c2b_line_convert_gtf_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src, ssize_t src_size);
- static inline void c2b_line_convert_gtf_to_bed(c2b_gtf_t g, char *dest_line, ssize_t *dest_size);
+ static inline void c2b_line_convert_gtf_ptr_to_bed(c2b_gtf_t *g, char *dest_line, ssize_t *dest_size);
+ static void c2b_gff_init_element(c2b_gff_t **e);
+ static void c2b_gff_delete_element(c2b_gff_t *e);
+ static void c2b_line_convert_gff_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src, ssize_t src_size);
+ static inline void c2b_line_convert_gff_ptr_to_bed(c2b_gff_t *g, char *dest_line, ssize_t *dest_size);
static void c2b_line_convert_psl_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src, ssize_t src_size);
static inline void c2b_psl_blockSizes_to_ptr(char *s, uint64_t bc);
static inline void c2b_psl_tStarts_to_ptr(char *s, uint64_t bc);
static inline void c2b_line_convert_psl_to_bed(c2b_psl_t p, char *dest_line, ssize_t *dest_size);
+ static void c2b_rmsk_init_element(c2b_rmsk_t **e);
+ static void c2b_rmsk_delete_element(c2b_rmsk_t *e);
static void c2b_line_convert_rmsk_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src, ssize_t src_size);
- static inline void c2b_line_convert_rmsk_to_bed(c2b_rmsk_t r, char *dest_line, ssize_t *dest_size);
+ static inline void c2b_line_convert_rmsk_ptr_to_bed(c2b_rmsk_t *r, char *dest_line, ssize_t *dest_size);
static void c2b_line_convert_sam_to_bed_unsorted_without_split_operation(char *dest, ssize_t *dest_size, char *src, ssize_t src_size);
static void c2b_line_convert_sam_to_bed_unsorted_with_split_operation(char *dest, ssize_t *dest_size, char *src, ssize_t src_size);
static inline void c2b_sam_cigar_str_to_ops(char *s);
+ static void c2b_sam_init_element(c2b_sam_t **e);
+ static void c2b_sam_delete_element(c2b_sam_t *e);
static void c2b_sam_init_cigar_ops(c2b_cigar_t **c, const ssize_t size);
+ static void c2b_sam_resize_cigar_ops(c2b_cigar_t **new_c, c2b_cigar_t *old_c);
static void c2b_sam_debug_cigar_ops(c2b_cigar_t *c);
static void c2b_sam_delete_cigar_ops(c2b_cigar_t *c);
- static inline void c2b_line_convert_sam_to_bed(c2b_sam_t s, char *dest_line, ssize_t *dest_size);
+ static inline void c2b_line_convert_sam_ptr_to_bed(c2b_sam_t *s, char *dest_line, ssize_t *dest_size, boolean print_modified_qname);
static void c2b_line_convert_vcf_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src, ssize_t src_size);
static inline boolean c2b_vcf_allele_is_id(char *s);
static inline boolean c2b_vcf_record_is_snv(char *ref, char *alt);
static inline boolean c2b_vcf_record_is_insertion(char *ref, char *alt);
static inline boolean c2b_vcf_record_is_deletion(char *ref, char *alt);
- static inline void c2b_line_convert_vcf_to_bed(c2b_vcf_t v, char *dest_line, ssize_t *dest_size);
+ static void c2b_vcf_init_element(c2b_vcf_t **e);
+ static void c2b_vcf_delete_element(c2b_vcf_t *e);
+ static inline void c2b_line_convert_vcf_ptr_to_bed(c2b_vcf_t *v, char *dest_line, ssize_t *dest_size);
static void c2b_line_convert_wig_to_bed_unsorted(char *dest, ssize_t *dest_size, char *src, ssize_t src_size);
static void * c2b_read_bytes_from_stdin(void *arg);
static void * c2b_process_intermediate_bytes_by_lines(void *arg);
diff --git a/applications/bed/starch/src/starchcluster_sge.tcsh b/applications/bed/conversion/src/wrappers/bam2bed_slurm
similarity index 60%
copy from applications/bed/starch/src/starchcluster_sge.tcsh
copy to applications/bed/conversion/src/wrappers/bam2bed_slurm
index e8321f5..b5fce5a 100755
--- a/applications/bed/starch/src/starchcluster_sge.tcsh
+++ b/applications/bed/conversion/src/wrappers/bam2bed_slurm
@@ -1,12 +1,8 @@
#!/bin/tcsh -ef
-# author : sjn and apr
-# date : Feb 2012
-# version : v2.5.0
-
-#
-# BEDOPS
-# Copyright (C) 2011-2015 Shane Neph, Scott Kuehn and Alex Reynolds
+# bam2bed_slurm
+# -----------------------------------------------------------------------
+# Copyright (C) 2011-2016 Shane Neph and Alex Reynolds
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -21,31 +17,25 @@
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
####################################################
# cluster variables:
# change to match your environment
-# may also require changes to 2 'qsub' calls below
####################################################
-set shell = "-S /bin/tcsh"
-set queue = "-q all.q"
-set misc_opts = "-V -cwd -w e -r yes -now no"
-set soundoff = "-j n -e /dev/null -o /dev/null"
-set sge_opts = "$queue $shell $misc_opts $soundoff"
+set slurm_opts = "-D $PWD -o /dev/null -e /dev/null"
############################
# some input error checking
############################
-set help = "\nUsage: starchcluster_sge [--help] [--clean] <input-bed-file> [output-starch-file]\n\n"
-set help = "$help Pass in the name of a BED file to create a starch archive using the cluster.\n\n"
-set help = "$help (stdin isn't supported through this wrapper script, but starch supports it natively.)\n\n"
-set help = "$help Add --clean to remove <input-bed-file> after starching it up.\n\n"
-set help = "$help You can pass in the name of the output starch archive to be created.\n"
+set help = "\nUsage: bam2bed_slurm [--help] [--clean] <input-indexed-bam-file> [output-bed-file]\n\n"
+set help = "$help Pass in the name of an indexed BAM file to create a sorted BED file using the cluster.\n\n"
+set help = "$help (stdin isn't supported through this wrapper script.)\n\n"
+set help = "$help Add --clean to remove <input-indexed-bam-file> after turning it into BED.\n\n"
+set help = "$help You can pass in the name of the output bed archive to be created.\n"
set help = "$help Otherwise, the output will have the same name as the input file, with an additional\n"
-set help = "$help '.starch' ending. If the input file ends with '.bed', that will be stripped off.\n"
+set help = "$help '.bed' ending. If the input file ends with '.bam', that will be stripped off.\n"
if ( $#argv == 0 ) then
printf "$help"
@@ -65,7 +55,7 @@ foreach argc (`seq 1 $#argv`)
set output = "$argv[$argc]"
else
set originput = "$argv[$argc]"
- set output = $originput:t.starch
+ set output = $originput:t.bed
endif
@ inputset = 1
else if ( $inputset > 0 ) then
@@ -74,7 +64,7 @@ foreach argc (`seq 1 $#argv`)
exit -1
else
set originput = "$argv[$argc]"
- set output = $originput:t.starch
+ set output = $originput:t.bed
@ inputset = 1
endif
end
@@ -83,17 +73,23 @@ if ( $inputset == 0 ) then
printf "No input file specified\n"
exit -1
else if ( ! -s $originput ) then
- printf "Unable to find file: %s\n" $originput
+ printf "Unable to find BAM file: %s\n" $originput
+ exit -1
+else if ( "$output" == "$originput:t.bed" && "$originput:e" == "bam" ) then
+ set output = "$originput:t:r.bed"
+endif
+
+set origininputindex = "$originput.bai"
+if ( ! -s $origininputindex ) then
+ printf "Unable to find associated BAI file (is the BAM file indexed?): %s\n" $origininputindex
exit -1
-else if ( "$output" == "$originput:t.starch" && "$originput:e" == "bed" ) then
- set output = "$originput:t:r.starch"
endif
###############################################################
# new working directory to keep file pileups local to this job
###############################################################
-set nm = scs.`uname -a | cut -f2 -d' '`.$$
+set nm = b2scs.`uname -a | cut -f2 -d' '`.$$
if ( -d $nm ) then
rm -rf $nm
endif
@@ -119,40 +115,31 @@ else
endif
#####################################################
-# extract information by chromosome and starch it up
+# extract information by chromosome and bam2bed it
#####################################################
set files = ()
set jids = ()
@ cntr = 0
-foreach chrom (`bedextract --list-chr $input`)
-
-qsub $sge_opts -N $nm.$cntr > /dev/stderr << __EXTRACTION__
- bedextract $chrom $input | starch - > $cntr
-__EXTRACTION__
-
- set jids = ($jids $nm.$cntr)
- set files = ($files $cntr)
+foreach chrom (`samtools idxstats $input | awk '$1!~"^*"'`)
+ set res = `sbatch $slurm_opts -J "$nm.$cntr" --wrap="module add samtools; module add bedops; samtools view -b $input $chrom | bam2bed - > $here/$nm/$cntr"`
+ set slurm_jid = `echo $res | sed 's|^Submitted batch job ||'`
+ set jids = ($jids $slurm_jid)
+ set files = ($files $here/$nm/$cntr)
@ cntr++
end
if ( $cntr == 0 ) then
- printf "Program problem: no files were submitted to the cluster?\n"
+ printf "Program problem: no files were submitted to the cluster? If samtools is not found, please use 'module add samtools' or similar.\n"
+ rm -rf $nm
exit -1
endif
##################################################
-# create final starch archive and clean things up
+# create final bed archive and clean things up
##################################################
-qsub $sge_opts -N $nm.union -hold_jid `echo $jids | tr ' ' ','` > /dev/stderr << __CATTED__
- starchcat $files > $output
- cd $here
- rm -rf $nm
-
- if ( $clean > 0 ) then
- rm -f $originput
- endif
-__CATTED__
+set dependencies = `echo $jids | tr ' ' ':'`
+sbatch $slurm_opts -J $nm.union --dependency=afterok:$dependencies --wrap="module add bedops; bedops --everything $files > $output; cd $here; rm -rf $nm; if [ $clean != 0 ]; then rm -f $originput; fi;"
exit 0
diff --git a/applications/bed/starch/src/starchcluster_sge.tcsh b/applications/bed/conversion/src/wrappers/bam2starch_slurm
similarity index 60%
copy from applications/bed/starch/src/starchcluster_sge.tcsh
copy to applications/bed/conversion/src/wrappers/bam2starch_slurm
index e8321f5..b553fe0 100755
--- a/applications/bed/starch/src/starchcluster_sge.tcsh
+++ b/applications/bed/conversion/src/wrappers/bam2starch_slurm
@@ -1,12 +1,8 @@
#!/bin/tcsh -ef
-# author : sjn and apr
-# date : Feb 2012
-# version : v2.5.0
-
-#
-# BEDOPS
-# Copyright (C) 2011-2015 Shane Neph, Scott Kuehn and Alex Reynolds
+# bam2bed_slurm
+# -----------------------------------------------------------------------
+# Copyright (C) 2011-2016 Shane Neph and Alex Reynolds
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -21,31 +17,25 @@
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
####################################################
# cluster variables:
# change to match your environment
-# may also require changes to 2 'qsub' calls below
####################################################
-set shell = "-S /bin/tcsh"
-set queue = "-q all.q"
-set misc_opts = "-V -cwd -w e -r yes -now no"
-set soundoff = "-j n -e /dev/null -o /dev/null"
-set sge_opts = "$queue $shell $misc_opts $soundoff"
+set slurm_opts = "-D $PWD -o /dev/null -e /dev/null"
############################
# some input error checking
############################
-set help = "\nUsage: starchcluster_sge [--help] [--clean] <input-bed-file> [output-starch-file]\n\n"
-set help = "$help Pass in the name of a BED file to create a starch archive using the cluster.\n\n"
-set help = "$help (stdin isn't supported through this wrapper script, but starch supports it natively.)\n\n"
-set help = "$help Add --clean to remove <input-bed-file> after starching it up.\n\n"
-set help = "$help You can pass in the name of the output starch archive to be created.\n"
+set help = "\nUsage: bam2bed_slurm [--help] [--clean] <input-indexed-bam-file> [output-bed-file]\n\n"
+set help = "$help Pass in the name of an indexed BAM file to create a sorted BED file using the cluster.\n\n"
+set help = "$help (stdin isn't supported through this wrapper script.)\n\n"
+set help = "$help Add --clean to remove <input-indexed-bam-file> after turning it into BED.\n\n"
+set help = "$help You can pass in the name of the output bed archive to be created.\n"
set help = "$help Otherwise, the output will have the same name as the input file, with an additional\n"
-set help = "$help '.starch' ending. If the input file ends with '.bed', that will be stripped off.\n"
+set help = "$help '.bed' ending. If the input file ends with '.bam', that will be stripped off.\n"
if ( $#argv == 0 ) then
printf "$help"
@@ -65,7 +55,7 @@ foreach argc (`seq 1 $#argv`)
set output = "$argv[$argc]"
else
set originput = "$argv[$argc]"
- set output = $originput:t.starch
+ set output = $originput:t.bed
endif
@ inputset = 1
else if ( $inputset > 0 ) then
@@ -74,7 +64,7 @@ foreach argc (`seq 1 $#argv`)
exit -1
else
set originput = "$argv[$argc]"
- set output = $originput:t.starch
+ set output = $originput:t.bed
@ inputset = 1
endif
end
@@ -83,17 +73,23 @@ if ( $inputset == 0 ) then
printf "No input file specified\n"
exit -1
else if ( ! -s $originput ) then
- printf "Unable to find file: %s\n" $originput
+ printf "Unable to find BAM file: %s\n" $originput
+ exit -1
+else if ( "$output" == "$originput:t.bed" && "$originput:e" == "bam" ) then
+ set output = "$originput:t:r.bed"
+endif
+
+set origininputindex = "$originput.bai"
+if ( ! -s $origininputindex ) then
+ printf "Unable to find associated BAI file (is the BAM file indexed?): %s\n" $origininputindex
exit -1
-else if ( "$output" == "$originput:t.starch" && "$originput:e" == "bed" ) then
- set output = "$originput:t:r.starch"
endif
###############################################################
# new working directory to keep file pileups local to this job
###############################################################
-set nm = scs.`uname -a | cut -f2 -d' '`.$$
+set nm = b2scs.`uname -a | cut -f2 -d' '`.$$
if ( -d $nm ) then
rm -rf $nm
endif
@@ -119,40 +115,31 @@ else
endif
#####################################################
-# extract information by chromosome and starch it up
+# extract information by chromosome and bam2bed it
#####################################################
set files = ()
set jids = ()
@ cntr = 0
-foreach chrom (`bedextract --list-chr $input`)
-
-qsub $sge_opts -N $nm.$cntr > /dev/stderr << __EXTRACTION__
- bedextract $chrom $input | starch - > $cntr
-__EXTRACTION__
-
- set jids = ($jids $nm.$cntr)
- set files = ($files $cntr)
+foreach chrom (`samtools idxstats $input | awk '$1!~"^*"'`)
+ set res = `sbatch $slurm_opts -J "$nm.$cntr" --wrap="module add samtools; module add bedops; samtools view -b $input $chrom | bam2starch - > $here/$nm/$cntr"`
+ set slurm_jid = `echo $res | sed 's|^Submitted batch job ||'`
+ set jids = ($jids $slurm_jid)
+ set files = ($files $here/$nm/$cntr)
@ cntr++
end
if ( $cntr == 0 ) then
- printf "Program problem: no files were submitted to the cluster?\n"
+ printf "Program problem: no files were submitted to the cluster? If samtools is not found, please use 'module add samtools' or similar.\n"
+ rm -rf $nm
exit -1
endif
##################################################
-# create final starch archive and clean things up
+# create final bed archive and clean things up
##################################################
-qsub $sge_opts -N $nm.union -hold_jid `echo $jids | tr ' ' ','` > /dev/stderr << __CATTED__
- starchcat $files > $output
- cd $here
- rm -rf $nm
-
- if ( $clean > 0 ) then
- rm -f $originput
- endif
-__CATTED__
+set dependencies = `echo $jids | tr ' ' ':'`
+sbatch $slurm_opts -J $nm.union --dependency=afterok:$dependencies --wrap="module add bedops; starchcat $files > $output; cd $here; rm -rf $nm; if [ $clean != 0 ]; then rm -f $originput; fi;"
exit 0
diff --git a/applications/bed/sort-bed/src/CheckSort.cpp b/applications/bed/sort-bed/src/CheckSort.cpp
index 093c071..09a9d77 100644
--- a/applications/bed/sort-bed/src/CheckSort.cpp
+++ b/applications/bed/sort-bed/src/CheckSort.cpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/applications/bed/sort-bed/src/Makefile b/applications/bed/sort-bed/src/Makefile
index 54b5c8f..699a0b6 100644
--- a/applications/bed/sort-bed/src/Makefile
+++ b/applications/bed/sort-bed/src/Makefile
@@ -45,7 +45,7 @@ else
WARNINGS += -U__STRICT_ANSI__
endif
-build: $(BINDIR)/$(PROG) $(dependencies)
+build: $(BINDIR)/$(PROG) $(dependencies) update-sort-bed-slurm update-sort-bed-starch-slurm update-sort-bed-migrate-candidates
build_gprof: $(BINDIR)/gprof.$(PROG)
@@ -53,6 +53,15 @@ build_debug: $(BINDIR)/debug.$(PROG)
dependencies: $(dependencies)
+update-sort-bed-slurm:
+ mkdir -p $(BINDIR) && cp update-sort-bed-slurm.py $(BINDIR)/update-sort-bed-slurm
+
+update-sort-bed-starch-slurm:
+ mkdir -p $(BINDIR) && cp update-sort-bed-starch-slurm.py $(BINDIR)/update-sort-bed-starch-slurm
+
+update-sort-bed-migrate-candidates:
+ mkdir -p $(BINDIR) && cp update-sort-bed-migrate-candidates.py $(BINDIR)/update-sort-bed-migrate-candidates
+
$(BINDIR)/$(PROG) : $(dependencies)
mkdir -p $(BINDIR) && $(CXX) -o $@ $(FLAGS) $^ ${LIBRARIES}
@@ -75,3 +84,6 @@ clean:
rm -rf ${OBJDIR}
rm -f $(BINDIR)/$(PROG)
rm -f $(BINDIR)/*.$(PROG)
+ rm -f $(BINDIR)/update-sort-bed-slurm
+ rm -f $(BINDIR)/update-sort-bed-starch-slurm
+ rm -f $(BINDIR)/update-sort-bed-migrate-candidates
diff --git a/applications/bed/sort-bed/src/Makefile.darwin b/applications/bed/sort-bed/src/Makefile.darwin
index 3a4657f..6717c8a 100644
--- a/applications/bed/sort-bed/src/Makefile.darwin
+++ b/applications/bed/sort-bed/src/Makefile.darwin
@@ -35,7 +35,16 @@ BLDFLAGS = ${WARNINGS} ${OPTIMIZE}
INCLUDES = -iquote$(HEAD) -I${LOCALJANSSONINCDIR} -I${LOCALBZIP2INCDIR} -I${LOCALZLIBINCDIR}
STARCHOBJS = $(OBJ_DIR)/starchConstants.o $(OBJ_DIR)/starchFileHelpers.o $(OBJ_DIR)/starchHelpers.o $(OBJ_DIR)/starchMetadataHelpers.o $(OBJ_DIR)/unstarchHelpers.o $(OBJ_DIR)/starchSha1Digest.o $(OBJ_DIR)/starchBase64Coding.o
-build: sort
+build: sort update-sort-bed-slurm update-sort-bed-starch-slurm update-sort-bed-migrate-candidates
+
+update-sort-bed-slurm:
+ cp update-sort-bed-slurm.py ${DIST_DIR}/update-sort-bed-slurm
+
+update-sort-bed-starch-slurm:
+ cp update-sort-bed-starch-slurm.py ${DIST_DIR}/update-sort-bed-starch-slurm
+
+update-sort-bed-migrate-candidates:
+ cp update-sort-bed-migrate-candidates.py ${DIST_DIR}/update-sort-bed-migrate-candidates
sort: sortbuild
${CXX} -o ${DIST_DIR}/${PROG}_${ARCH} ${BLDFLAGS} ${LIBLOCATION} ${INCLUDES} -mmacosx-version-min=$(MIN_OSX_VERSION) -arch $(ARCH) -lc++ ${STARCHOBJS} ${OBJ_DIR}/SortDetails.o ${OBJ_DIR}/Sort.o ${OBJ_DIR}/CheckSort.o ${LIBRARIES}
diff --git a/applications/bed/sort-bed/src/Sort.cpp b/applications/bed/sort-bed/src/Sort.cpp
index 3c1a69d..5fb4805 100644
--- a/applications/bed/sort-bed/src/Sort.cpp
+++ b/applications/bed/sort-bed/src/Sort.cpp
@@ -5,7 +5,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/applications/bed/sort-bed/src/SortDetails.cpp b/applications/bed/sort-bed/src/SortDetails.cpp
index 15e9b98..18fb94b 100644
--- a/applications/bed/sort-bed/src/SortDetails.cpp
+++ b/applications/bed/sort-bed/src/SortDetails.cpp
@@ -5,7 +5,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -22,6 +22,7 @@
// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
//
+#include <algorithm>
#include <cinttypes>
#include <cerrno>
#include <cstdio>
@@ -390,9 +391,10 @@ mergeSort(FILE* output, FILE **tmpFiles, unsigned int numFiles)
/* error checking in processData() has already been performed, headers and empty rows removed, etc. */
unsigned int i = 0U;
int done = 0, currMin = 0, val = 0;
+ char* currRest = NULL;
char **chroms = static_cast<char**>( malloc(sizeof(char*) * static_cast<size_t>(numFiles)) );
-
- if(chroms == NULL)
+
+ if(chroms == NULL)
return -1;
Bed::SignedCoordType *starts = static_cast<Bed::SignedCoordType*>( malloc(sizeof(Bed::SignedCoordType) * static_cast<size_t>(numFiles)) );
@@ -419,6 +421,7 @@ mergeSort(FILE* output, FILE **tmpFiles, unsigned int numFiles)
rests[i] = static_cast<char*>( malloc(sizeof(char) * (BED_LINE_LEN+1)) );
if(rests[i] == NULL)
return -1;
+ rests[i][0] = '\0';
} /* for */
for(i=0; i < numFiles; ++i)
@@ -437,18 +440,32 @@ mergeSort(FILE* output, FILE **tmpFiles, unsigned int numFiles)
while(!done)
{
currMin = -1;
+ currRest = NULL;
for(i=0; i < numFiles; ++i)
{
if(starts[i]>=0)
{
if(currMin<0 || (val = strcmp(chroms[i], chroms[currMin]))<0)
- currMin = static_cast<int>(i);
+ currMin = static_cast<int>(i), currRest = rests[i];
else if(0 == val)
{
if(starts[i] < starts[currMin])
currMin = static_cast<int>(i);
else if(starts[i] == starts[currMin] && ends[i] < ends[currMin])
currMin = static_cast<int>(i);
+ else if(starts[i] == starts[currMin] && ends[i] == ends[currMin])
+ {
+ if (currRest == NULL)
+ {
+ currMin = static_cast<int>(i);
+ currRest = rests[i];
+ }
+ else if (strcmp(rests[i], currRest) < 0)
+ {
+ currMin = static_cast<int>(i);
+ currRest = rests[i];
+ }
+ }
}
}
} /* for */
@@ -462,6 +479,7 @@ mergeSort(FILE* output, FILE **tmpFiles, unsigned int numFiles)
fprintf(output, "%s\t%" PRId64 "\t%" PRId64 "%s\n", chroms[currMin],
starts[currMin], ends[currMin], rests[currMin]);
+ rests[currMin][0] = '\0';
fields[currMin] = fscanf(tmpFiles[currMin], "%s\t%" SCNd64 "\t%" SCNd64 "%[^\n]s\n",
chroms[currMin], &starts[currMin],
&ends[currMin], rests[currMin]);
@@ -1108,7 +1126,7 @@ printBed(FILE *out, BedData *beds)
if(beds == NULL)
return;
- for(i = 0; i < beds->numChroms; i++)
+ for(i = 0; i < beds->numChroms; i++)
for(j = 0; j < beds->chroms[i]->numCoords; j++)
{
fprintf(out, "%s\t%" PRId64 "\t%" PRId64, beds->chroms[i]->chromName, beds->chroms[i]->coords[j].startCoord,
@@ -1171,9 +1189,9 @@ lexSortBedData(BedData *beds)
}
/* sort coords */
- for(i = 0; i < beds->numChroms; ++i)
- {
- qsort(beds->chroms[i]->coords, static_cast<size_t>(beds->chroms[i]->numCoords), sizeof(BedCoordData), numCompareBedData);
+ for(i = 0; i < beds->numChroms; ++i)
+ {
+ std::sort(beds->chroms[i]->coords, (beds->chroms[i]->coords+static_cast<size_t>(beds->chroms[i]->numCoords)));
}
/* sort chroms */
@@ -1181,22 +1199,10 @@ lexSortBedData(BedData *beds)
return;
}
-int
-numCompareBedData(const void *pos1, const void *pos2)
-{
- Bed::SignedCoordType diff = (static_cast<const BedCoordData *>(pos1))->startCoord - (static_cast<const BedCoordData *>(pos2))->startCoord;
- if(diff)
- {
- return (diff > 0) ? 1 : -1;
- }
- diff = (static_cast<const BedCoordData *>(pos1))->endCoord - (static_cast<const BedCoordData *>(pos2))->endCoord;
- return (diff > 0) ? 1 : ((diff < 0) ? -1 : 0);
-}
-
int
lexCompareBedData(const void *chrPos1, const void *chrPos2)
{
- ChromBedData **chrPos1Cbd = static_cast<ChromBedData **>(const_cast<void *>(chrPos1));
- ChromBedData **chrPos2Cbd = static_cast<ChromBedData **>(const_cast<void *>(chrPos2));
+ ChromBedData* const* chrPos1Cbd = static_cast<ChromBedData* const*>(chrPos1);
+ ChromBedData* const* chrPos2Cbd = static_cast<ChromBedData* const*>(chrPos2);
return strcmp((*chrPos1Cbd)->chromName, (*chrPos2Cbd)->chromName);
}
diff --git a/applications/bed/sort-bed/src/Structures.hpp b/applications/bed/sort-bed/src/Structures.hpp
index 6bf6048..7d6c1f2 100644
--- a/applications/bed/sort-bed/src/Structures.hpp
+++ b/applications/bed/sort-bed/src/Structures.hpp
@@ -5,7 +5,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -39,16 +39,55 @@ static const unsigned long NUM_CHROM_EST = 32;
#define GT(A,B) ((A) > (B) ? 1 : 0)
/* Data Structures */
-typedef struct {
+struct BedCoordData;
+int bcd_cmp(BedCoordData const& b1, BedCoordData const& b2);
+
+struct BedCoordData {
Bed::SignedCoordType startCoord;
Bed::SignedCoordType endCoord;
char *data;
-} BedCoordData;
+
+ inline friend
+ int bcd_cmp(BedCoordData const& b1, BedCoordData const& b2)
+ {
+ Bed::SignedCoordType diff = (b1.startCoord - b2.startCoord);
+ if(diff)
+ {
+ return (diff > 0) ? 1 : -1;
+ }
+ diff = (b1.endCoord - b2.endCoord);
+ if(diff)
+ {
+ return (diff > 0) ? 1 : -1;
+ }
+
+ // so far equivalent, check data member for differences
+ if(b1.data != NULL)
+ {
+ if(b2.data != NULL)
+ {
+ return strcmp(b1.data, b2.data);
+ }
+ else
+ {
+ return 1;
+ }
+ }
+ else if(b2.data != NULL)
+ {
+ return -1;
+ }
+ return 0;
+ }
+
+ inline friend bool operator<(BedCoordData const& b1, BedCoordData const& b2) { return bcd_cmp(b1,b2) < 0; }
+ inline friend bool operator>(BedCoordData const& b1, BedCoordData const& b2) { return bcd_cmp(b1,b2) > 0; }
+ inline friend bool operator==(BedCoordData const& b1, BedCoordData const& b2) { return bcd_cmp(b1,b2) == 0; }
+};
typedef struct {
char chromName[CHROM_NAME_LEN + 1];
Bed::LineCountType numCoords;
- //BedCoordData coords[MAX_BED_ITEMS];
BedCoordData *coords;
} ChromBedData;
@@ -101,6 +140,4 @@ numCompareBedData(const void *pos1, const void *pos2);
int
lexCompareBedData(const void *pos1, const void *pos2);
-
-
#endif /* BED_STRUCTURES_H */
diff --git a/applications/bed/sort-bed/src/update-sort-bed-migrate-candidates.py b/applications/bed/sort-bed/src/update-sort-bed-migrate-candidates.py
new file mode 100755
index 0000000..9b1ad28
--- /dev/null
+++ b/applications/bed/sort-bed/src/update-sort-bed-migrate-candidates.py
@@ -0,0 +1,613 @@
+#!/usr/bin/env python
+
+#
+# BEDOPS
+# Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import sys
+import os
+import argparse
+import errno
+import subprocess
+import json
+import logging
+
+name = "update-sort-bed-migrate-candidates"
+citation = " citation: http://bioinformatics.oxfordjournals.org/content/28/14/1919.abstract"
+authors = " authors: Alex Reynolds and Shane Neph"
+version = " version: 2.4.26"
+usage = """ $ update-sort-bed-migrate-candidates [ --dry-run ] [ --debug ]
+ [ --write-list |
+ --resort-immediately |
+ --resort-in-parallel-via-slurm
+ [ --slurm-memory <MB> ]
+ [ --slurm-partition <SLURM partition> ]
+ [ --slurm-workdir <working directory> ]
+ [ --slurm-output <SLURM output directory> ]
+ [ --slurm-error <SLURM error directory> ]
+ ]
+ [ --bedops-root-dir <bedops directory> ]
+ [ --bedextract-path <path> ]
+ [ --sort-bed-path <path> ]
+ [ --unstarch-path <path> ]
+ [ --starch-path <path> ]
+ [ --starchcat-path <path> ]
+ [ --update-sort-bed-slurm-path <path> ]
+ [ --update-sort-bed-starch-slurm-path <path> ]
+ --parent-dir <parent directory>
+ [ --non-recursive-search ]"""
+help = """
+ The "update-sort-bed-migrate-candidates" utility recursively locates BED and
+ Starch files in the specified parent directory and tests if they require
+ re-sorting to conform to the updated, post-v2.4.20 "sort-bed" order.
+
+ Files with the extensions starch, bstarch, gstarch, bed, or bed[g|G]raph in
+ the parent directory are tested. Files without these extensions are ignored.
+
+ If the "--non-recursive-search" option is added, this utility will only search
+ within the specified parent directory, and go no deeper.
+
+ This utility offers one of three (exclusive) actions for migration:
+
+ 1. Using "--write-list", files that require re-sorting can have their paths
+ written to standard output, which can be written to a file to be processed
+ later on, as desired.
+
+ 2. Using "--resort-immediately", qualifying files can be resorted immediately
+ after all candidates are found, through a local, serial application of
+ 'sort-bed'.
+
+ 3. Using "--resort-in-parallel-via-slurm", candidate files can be migrated by
+ applying the 'update-sort-bed-slurm' script to resort in parallel on a
+ computational cluster managed with a SLURM job scheduler.
+
+ Note that one of these three options must be chosen, to perform the
+ stated action, and only one option can be selected.
+
+ When using "--resort-immediately" or "--resort-in-parallel-via-slurm", the
+ resorted files will have the name of the original BED or Starch file. The
+ original files will have their old name, with the ".backup" extension.
+
+ Use the "--bedops-root-dir" option to specify the directory containing the
+ BEDOPS toolkit binaries to be used for migration. You can provide more specific
+ paths to individual binaries, using the following options:
+
+ Add the "--bedextract-path", "--sort-bed-path", "--unstarch-path",
+ "--starch-path", "--starchcat-path", "--update-sort-bed-slurm", and/or
+ "--update-sort-bed-starch-slurm" options to specify custom paths to versions
+ of these tools, if desired. These values will be passed along to downstream
+ helper scripts that use them.
+
+ ---
+
+ Suggestions:
+
+ 1. Add the "--dry-run" option to "--resort-immediately" or the
+ "--resort-in-parallel-via-slurm" options to see the behavior before any
+ filesystem actions are performed. Remove "--dry-run" to perform the
+ specified work.
+
+ 2. If you use the resort-via-SLURM option, consider using "--slurm-memory",
+ "--slurm-partition", "--slurm-workdir", "--slurm-output", and
+ "--slurm-error" options to match the setup of your particular cluster
+ environment and inputs.
+
+ 3. Add "--debug" option to log debug statements to get more detail about
+ internal operation of update process.
+
+"""
+
+def main():
+ parser = argparse.ArgumentParser(prog=name, usage=usage, add_help=False)
+ parser.add_argument('--help', '-h', action='store_true', dest='help')
+ parser.add_argument('--write-list', '-l', action="store_true", dest='write_list')
+ parser.add_argument('--resort-immediately', '-i', action="store_true", dest='resort_immediately')
+ parser.add_argument('--resort-in-parallel-via-slurm', '-s', action="store_true", dest='resort_in_parallel_via_slurm')
+ parser.add_argument('--slurm-memory', '-m', type=str, action="store", dest='slurm_memory')
+ parser.add_argument('--slurm-partition', '-p', type=str, action="store", dest='slurm_partition')
+ parser.add_argument('--slurm-workdir', '-w', type=str, action="store", dest='slurm_workdir')
+ parser.add_argument('--slurm-output', '-r', type=str, action="store", dest='slurm_output')
+ parser.add_argument('--slurm-error', '-e', type=str, action="store", dest='slurm_error')
+ parser.add_argument('--bedops-root-dir', '-b', type=str, action="store", dest='bedops_root_dir')
+ parser.add_argument('--bedextract-path', '-x', type=str, action="store", dest='bedextract_path')
+ parser.add_argument('--sort-bed-path', '-o', type=str, action="store", dest='sort_bed_path')
+ parser.add_argument('--unstarch-path', '-u', type=str, action="store", dest='unstarch_path')
+ parser.add_argument('--starch-path', '-t', type=str, action="store", dest='starch_path')
+ parser.add_argument('--starchcat-path', '-z', type=str, action="store", dest='starchcat_path')
+ parser.add_argument('--update-sort-bed-slurm-path', '-y', type=str, action="store", dest='update_sort_bed_slurm_path')
+ parser.add_argument('--update-sort-bed-starch-slurm-path', '-q', type=str, action="store", dest='update_sort_bed_starch_slurm_path')
+ parser.add_argument('--dry-run', '-n', action="store_true", dest='dry_run')
+ parser.add_argument('--debug', '-d', action="store_true", dest='debug')
+ parser.add_argument('--parent-dir', '-a', type=str, action="store", dest='parent_dir')
+ parser.add_argument('--non-recursive-search', '-v', action="store_true", dest='non_recursive_search')
+ args = parser.parse_args()
+
+ action_counter = 0
+ if args.write_list:
+ action_counter += 1
+ if args.resort_immediately:
+ action_counter += 1
+ if args.resort_in_parallel_via_slurm:
+ action_counter += 1
+
+ if args.help or not args.parent_dir or action_counter != 1:
+ sys.stdout.write(name + '\n')
+ sys.stdout.write(citation + '\n')
+ sys.stdout.write(version + '\n')
+ sys.stdout.write(authors + '\n\n')
+ sys.stdout.write(usage + '\n')
+ sys.stdout.write(help)
+ if args.help:
+ sys.exit(os.EX_OK)
+ else:
+ sys.exit(errno.EINVAL)
+
+ logger = None
+ if args.debug:
+ logging.basicConfig(level=logging.DEBUG)
+ logger = logging.getLogger(__name__)
+
+ bedextract_path = None
+ if logger: logger.info('Locating \"bedextract\" binary')
+ if not args.bedextract_path and not args.bedops_root_dir:
+ bedextract_path = find_binary('bedextract')
+ if not bedextract_path:
+ if logger: logger.info('Could not locate \"bedextract\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS bedextract\n")
+ sys.exit(errno.EEXIST)
+ elif args.bedops_root_dir and not args.bedextract_path:
+ bedextract_path = os.path.join(args.bedops_root_dir, 'bedextract')
+ elif args.bedextract_path:
+ bedextract_path = args.bedextract_path
+ elif not cmd_exists('bedextract'):
+ if logger: logger.info('Could not locate \"bedextract\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS bedextract\n")
+ sys.exit(errno.EEXIST)
+ if logger: logger.info('Location of \"bedextract\" is set to [%s]' % (bedextract_path))
+
+ sort_bed_path = None
+ if logger: logger.info('Locating \"sort-bed\" binary')
+ if not args.sort_bed_path and not args.bedops_root_dir:
+ sort_bed_path = find_binary('sort-bed')
+ if not sort_bed_path:
+ if logger: logger.info('Could not locate \"sort-bed\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS sort-bed\n")
+ sys.exit(errno.EEXIST)
+ elif args.bedops_root_dir and not args.sort_bed_path:
+ sort_bed_path = os.path.join(args.bedops_root_dir, 'sort-bed')
+ elif args.sort_bed_path:
+ sort_bed_path = args.sort_bed_path
+ elif not cmd_exists('sort-bed'):
+ if logger: logger.info('Could not locate \"sort-bed\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS sort-bed\n")
+ sys.exit(errno.EEXIST)
+ if logger: logger.info('Location of \"sort-bed\" is set to [%s]' % (sort_bed_path))
+
+ unstarch_path = None
+ if logger: logger.info('Locating \"unstarch\" binary')
+ if not args.unstarch_path and not args.bedops_root_dir:
+ unstarch_path = find_binary('unstarch')
+ if not unstarch_path:
+ if logger: logger.info('Could not locate \"unstarch\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS unstarch\n")
+ sys.exit(errno.EEXIST)
+ elif args.bedops_root_dir and not args.unstarch_path:
+ unstarch_path = os.path.join(args.bedops_root_dir, 'unstarch')
+ elif args.unstarch_path:
+ unstarch_path = args.unstarch_path
+ elif not cmd_exists('unstarch'):
+ if logger: logger.info('Could not locate \"unstarch\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS unstarch\n")
+ sys.exit(errno.EEXIST)
+ if logger: logger.info('Location of \"unstarch\" is set to [%s]' % (unstarch_path))
+
+ starch_path = None
+ if logger: logger.info('Locating \"starch\" binary')
+ if not args.starch_path and not args.bedops_root_dir:
+ starch_path = find_binary('starch')
+ if not starch_path:
+ if logger: logger.info('Could not locate \"starch\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS starch\n")
+ sys.exit(errno.EEXIST)
+ elif args.bedops_root_dir and not args.starch_path:
+ starch_path = os.path.join(args.bedops_root_dir, 'starch')
+ elif args.starch_path:
+ starch_path = args.starch_path
+ elif not cmd_exists('starch'):
+ if logger: logger.info('Could not locate \"starch\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS starch\n")
+ sys.exit(errno.EEXIST)
+ if logger: logger.info('Location of \"starch\" is set to [%s]' % (starch_path))
+
+ starchcat_path = None
+ if logger: logger.info('Locating \"starchcat\" binary')
+ if not args.starchcat_path and not args.bedops_root_dir:
+ starchcat_path = find_binary('starchcat')
+ if not starchcat_path:
+ if logger: logger.info('Could not locate \"starchcat\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS starchcat\n")
+ sys.exit(errno.EEXIST)
+ elif args.bedops_root_dir and not args.starchcat_path:
+ starchcat_path = os.path.join(args.bedops_root_dir, 'starchcat')
+ elif args.starchcat_path:
+ starchcat_path = args.starchcat_path
+ elif not cmd_exists('starchcat'):
+ if logger: logger.info('Could not locate \"starchcat\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS starchcat\n")
+ sys.exit(errno.EEXIST)
+ if logger: logger.info('Location of \"starchcat\" is set to [%s]' % (starchcat_path))
+
+ update_sort_bed_slurm_path = None
+ if logger: logger.info('Locating \"update-sort-bed-slurm\" script')
+ if not args.update_sort_bed_slurm_path and not args.bedops_root_dir:
+ update_sort_bed_slurm_path = find_binary('update-sort-bed-slurm')
+ if not update_sort_bed_slurm_path:
+ if logger: logger.info('Could not locate \"update-sort-bed-slurm\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS update-sort-bed-slurm\n")
+ sys.exit(errno.EEXIST)
+ elif args.bedops_root_dir and not args.update_sort_bed_slurm_path:
+ update_sort_bed_slurm_path = os.path.join(args.bedops_root_dir, 'update-sort-bed-slurm')
+ elif args.update_sort_bed_slurm_path:
+ update_sort_bed_slurm_path = args.update_sort_bed_slurm_path
+ elif not cmd_exists('update-sort-bed-slurm'):
+ if logger: logger.info('Could not locate \"update-sort-bed-slurm\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS update-sort-bed-slurm\n")
+ sys.exit(errno.EEXIST)
+ if logger: logger.info('Location of \"update-sort-bed-slurm\" is set to [%s]' % (update_sort_bed_slurm_path))
+
+ update_sort_bed_starch_slurm_path = None
+ if logger: logger.info('Locating \"update-sort-bed-starch-slurm\" script')
+ if not args.update_sort_bed_starch_slurm_path or not args.bedops_root_dir:
+ update_sort_bed_starch_slurm_path = find_binary('update-sort-bed-starch-slurm')
+ if not update_sort_bed_starch_slurm_path:
+ if logger: logger.info('Could not locate \"update-sort-bed-starch-slurm\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS update-sort-bed-starch-slurm\n")
+ sys.exit(errno.EEXIST)
+ elif args.bedops_root_dir and not args.update_sort_bed_starch_slurm_path:
+ update_sort_bed_starch_slurm_path = os.path.join(args.bedops_root_dir, 'update-sort-bed-starch-slurm')
+ elif args.update_sort_bed_starch_slurm_path:
+ update_sort_bed_starch_slurm_path = args.update_sort_bed_starch_slurm_path
+ elif not cmd_exists('update-sort-bed-starch-slurm'):
+ if logger: logger.info('Could not locate \"update-sort-bed-starch-slurm\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS update-sort-bed-starch-slurm\n")
+ sys.exit(errno.EEXIST)
+ if logger: logger.info('Location of \"update-sort-bed-starch-slurm\" is set to [%s]' % (update_sort_bed_starch_slurm_path))
+
+ bed_candidates = []
+ starch_candidates = []
+ if logger: logger.info('Initializing unfiltered BED and Starch candidate lists')
+
+ for root, directories, filenames in os.walk(args.parent_dir):
+ for filename in filenames:
+ candidate = os.path.join(root, filename)
+ if candidate.lower().endswith(('.starch', 'bstarch', 'gstarch')):
+ starch_candidates.append(candidate)
+ if candidate.lower().endswith(('.bed', '.bedgraph', '.bedGraph')):
+ bed_candidates.append(candidate)
+ if args.non_recursive_search: break
+ if logger: logger.info('Unfiltered BED candidates are [%s]' % (str(bed_candidates)))
+ if logger: logger.info('Unfiltered Starch candidates are [%s]' % (str(starch_candidates)))
+
+ filtered_bed_candidates = []
+ filtered_starch_candidates = []
+ if logger: logger.info('Initializing filtered BED and Starch candidate lists')
+
+ if len(bed_candidates) > 0:
+ # run sort-bed --check-sort on each BED candidate
+ for candidate in bed_candidates:
+ test_bed_sort_cmd_components = [
+ sort_bed_path,
+ '--check-sort',
+ candidate,
+ '>',
+ '/dev/null'
+ ]
+ if logger: logger.info('Checking BED candidate via [%s]' % (' '.join(test_bed_sort_cmd_components)))
+ test_bed_sort_process = subprocess.Popen(' '.join(test_bed_sort_cmd_components),
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ close_fds=True)
+ (test_bed_sort_stdout, test_bed_sort_stderr) = test_bed_sort_process.communicate()
+ if test_bed_sort_process.returncode != 0:
+ filtered_bed_candidates.append(candidate)
+ if logger: logger.info('Filtered BED candidates are [%s]' % (str(filtered_bed_candidates)))
+
+ if len(starch_candidates) > 0:
+ # run sort-bed --check-sort on the extracted output of a Starch candidate
+ for candidate in starch_candidates:
+ get_archive_version_cmd_components = [
+ unstarch_path,
+ '--list-json',
+ candidate
+ ]
+ try:
+ get_archive_version_cmd_result = subprocess.check_output(get_archive_version_cmd_components)
+ except subprocess.CalledProcessError as err:
+ get_archive_version_cmd_result = "ERROR: Command '{}' returned with error (code {}): {}".format(err.cmd, err.returncode, err.output)
+ raise
+ archive_metadata = json.loads(get_archive_version_cmd_result.decode('utf-8'))
+ try:
+ archive_version = archive_metadata['archive']['version']
+ except KeyError as err:
+ sys.stderr.write("ERROR: Could not read archive version from Starch metadata\n")
+ raise
+
+ if archive_version['major'] < 2 or (archive_version['major'] == 2 and archive_version['minor'] < 2):
+ test_starch_sort_cmd_components = [
+ unstarch_path,
+ candidate,
+ '|',
+ sort_bed_path,
+ '--check-sort',
+ '-',
+ '>',
+ '/dev/null'
+ ]
+ if logger: logger.info('Checking Starch candidate via [%s]' % (' '.join(test_starch_sort_cmd_components)))
+ test_starch_sort_process = subprocess.Popen(' '.join(test_starch_sort_cmd_components),
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ close_fds=True)
+ (test_starch_sort_stdout, test_starch_sort_stderr) = test_starch_sort_process.communicate()
+ if test_starch_sort_process.returncode != 0:
+ filtered_starch_candidates.append(candidate)
+ if logger: logger.info('Filtered Starch candidates are [%s]' % (str(filtered_starch_candidates)))
+
+ all_candidates = merge(filtered_bed_candidates, filtered_starch_candidates)
+ if logger: logger.info('Merged filtered BED and Starch candidate lists into all-candidate list')
+
+ if args.write_list:
+ if logger: logger.info('Writing candidate paths to standard output stream')
+ for candidate in all_candidates:
+ sys.stdout.write("%s\n" % (candidate))
+
+ if args.resort_immediately:
+ if logger: logger.info('Resorting candidates immediately')
+ for candidate in all_candidates:
+ #
+ # Filenames
+ #
+ src_original_fn = candidate
+ src_backup_fn = src_original_fn + '.backup'
+ dest_new_fn = src_original_fn + '.new'
+ dest_final_fn = src_original_fn
+ #
+ # BED
+ #
+ if candidate.lower().endswith(('.bed', '.bedgraph', '.bedGraph')):
+ if logger: logger.info('Planning to resort BED candidate [%s]' % (src_original_fn))
+ bed_resort_cmd_components = [
+ sort_bed_path,
+ src_original_fn,
+ '>',
+ dest_new_fn
+ ]
+ if logger: logger.info('Planning to resort BED candidate via [%s]' % (' '.join(bed_resort_cmd_components)))
+ if not args.dry_run:
+ bed_resort_process = subprocess.Popen(' '.join(bed_resort_cmd_components),
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ close_fds=True)
+ (bed_resort_stdout, bed_resort_stderr) = bed_resort_process.communicate()
+ if bed_resort_process.returncode != 0:
+ raise
+ else:
+ if logger: logger.info('Planning to rename [%s] to [%s]' % (src_original_fn, src_backup_fn))
+ try:
+ os.rename(src_original_fn, src_backup_fn)
+ if logger: logger.info('Renamed [%s] to [%s]' % (src_original_fn, src_backup_fn))
+ except OSError as err:
+ sys.stderr.write("Error: Could not rename [%s] to [%s]\n" % (src_original_fn, src_backup_fn))
+ raise
+ if logger: logger.info('Planning to rename [%s] to [%s]' % (dest_new_fn, dest_final_fn))
+ try:
+ os.rename(dest_new_fn, dest_final_fn)
+ if logger: logger.info('Renamed [%s] to [%s]' % (dest_new_fn, dest_final_fn))
+ except OSError as err:
+ sys.stderr.write("Error: Could not rename [%s] to [%s]\n" % (dest_new_fn, dest_final_fn))
+ raise
+ if logger: logger.info('Resorted BED candidate [%s] to [%s]' % (src_original_fn, dest_final_fn))
+ #
+ # Starch
+ #
+ if candidate.lower().endswith(('.starch', 'bstarch', 'gstarch')):
+ if logger: logger.info('Planning to resort Starch candidate [%s]' % (src_original_fn))
+ starch_resort_cmd_components = [
+ unstarch_path,
+ src_original_fn,
+ '|',
+ sort_bed_path,
+ '-',
+ '|',
+ starch_path,
+ '-',
+ '>',
+ dest_new_fn
+ ]
+ if logger: logger.info('Planning to resort Starch candidate via [%s]' % (' '.join(starch_resort_cmd_components)))
+ if not args.dry_run:
+ starch_resort_process = subprocess.Popen(' '.join(starch_resort_cmd_components),
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ close_fds=True)
+ (starch_resort_stdout, starch_resort_stderr) = starch_resort_process.communicate()
+ if starch_resort_process.returncode != 0:
+ raise
+ else:
+ if logger: logger.info('Debug: Planning to rename [%s] to [%s]' % (src_original_fn, src_backup_fn))
+ try:
+ os.rename(src_original_fn, src_backup_fn)
+ if logger: logger.info('Renamed [%s] to [%s]' % (src_original_fn, src_backup_fn))
+ except OSError as err:
+ sys.stderr.write("Error: Could not rename [%s] to [%s]\n" % (src_original_fn, src_backup_fn))
+ raise
+ sys.stderr.write("Debug: Planning to rename [%s] to [%s]\n" % (dest_new_fn, dest_final_fn))
+ try:
+ os.rename(dest_new_fn, dest_final_fn)
+ if logger: logger.info('Renamed [%s] to [%s]' % (dest_new_fn, dest_final_fn))
+ except OSError as err:
+ sys.stderr.write("Error: Could not rename [%s] to [%s]\n" % (dest_new_fn, dest_final_fn))
+ raise
+ if logger: logger.info('Resorted Starch candidate [%s] to [%s]' % (src_original_fn, dest_final_fn))
+
+ if args.resort_in_parallel_via_slurm:
+ if logger: logger.info('Resorting candidates in parallel via SLURM job scheduler')
+ for candidate in all_candidates:
+ #
+ # Filenames
+ #
+ src_original_fn = candidate
+ src_backup_fn = src_original_fn + '.backup'
+ dest_temp_fn = src_original_fn + '.temp'
+ dest_final_fn = src_original_fn
+ #
+ # BED
+ #
+ if candidate.lower().endswith(('.bed', '.bedgraph', '.bedGraph')):
+ if logger: logger.info('Planning to resort BED candidate [%s]' % (src_original_fn))
+ bed_slurm_resort_cmd_components = [
+ update_sort_bed_slurm_path,
+ '--input-original',
+ src_original_fn,
+ '--input-backup',
+ src_backup_fn,
+ '--output-temp',
+ dest_temp_fn,
+ '--output-final',
+ dest_final_fn,
+ '--bedextract-path',
+ bedextract_path,
+ '--sort-bed-path',
+ sort_bed_path
+ ]
+ slurm_options = customize_slurm_options(args)
+ if slurm_options:
+ bed_slurm_resort_cmd_components.append(slurm_options)
+ if args.debug:
+ bed_slurm_resort_cmd_components.append('--debug')
+ if logger: logger.info('Planning to resort BED candidate via [%s]' % (' '.join(bed_slurm_resort_cmd_components)))
+ if not args.dry_run:
+ bed_slurm_resort_process = subprocess.Popen(' '.join(bed_slurm_resort_cmd_components),
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ close_fds=True)
+ (bed_slurm_resort_stdout, bed_slurm_resort_stderr) = bed_slurm_resort_process.communicate()
+ if bed_slurm_resort_process.returncode != 0:
+ raise
+ if logger: logger.info('SLURM task submitted to resort [%s]' % (src_original_fn))
+
+ # Starch
+ if candidate.lower().endswith(('.starch', 'bstarch', 'gstarch')):
+ if logger: logger.info('Planning to resort Starch candidate [%s]' % (src_original_fn))
+ starch_slurm_resort_cmd_components = [
+ update_sort_bed_starch_slurm_path,
+ '--input-original',
+ src_original_fn,
+ '--input-backup',
+ src_backup_fn,
+ '--output-temp',
+ dest_temp_fn,
+ '--output-final',
+ dest_final_fn,
+ '--bedextract-path',
+ bedextract_path,
+ '--sort-bed-path',
+ sort_bed_path,
+ '--unstarch-path',
+ unstarch_path,
+ '--starch-path',
+ starch_path,
+ '--starchcat-path',
+ starchcat_path
+ ]
+ slurm_options = customize_slurm_options(args)
+ if slurm_options:
+ starch_slurm_resort_cmd_components.append(slurm_options)
+ if args.debug:
+ starch_slurm_resort_cmd_components.append('--debug')
+ if logger: logger.info('Planning to resort Starch candidate via [%s]' % (' '.join(starch_slurm_resort_cmd_components)))
+ if not args.dry_run:
+ starch_slurm_resort_process = subprocess.Popen(' '.join(starch_slurm_resort_cmd_components),
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ close_fds=True)
+ (starch_slurm_resort_stdout, starch_slurm_resort_stderr) = starch_slurm_resort_process.communicate()
+ if starch_slurm_resort_process.returncode != 0:
+ raise
+ if logger: logger.info('SLURM task submitted to resort [%s]' % (src_fn))
+
+def find_binary(binary_to_find):
+ for p in os.environ['PATH'].split(':'):
+ for r, d, f in os.walk(p):
+ for filename in f:
+ if filename == binary_to_find:
+ return os.path.join(r, filename)
+ return None
+
+def cmd_exists(cmd):
+ return subprocess.call("type " + cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0
+
+def customize_slurm_options(args):
+ custom_slurm_options = ""
+ if args.slurm_memory:
+ custom_slurm_options += "--slurm-memory " + args.slurm_memory
+ if args.slurm_partition:
+ custom_slurm_options += "--slurm-partition " + args.slurm_partition
+ if args.slurm_workdir:
+ custom_slurm_options += "--slurm-workdir " + args.slurm_workdir
+ if args.slurm_output:
+ custom_slurm_options += "--slurm-output " + args.slurm_output
+ if args.slurm_error:
+ custom_slurm_options += "--slurm-error " + args.slurm_error
+ return custom_slurm_options if len(custom_slurm_options) > 0 else None
+
+def merge(l, m):
+ result = []
+ i = j = 0
+ total = len(l) + len(m)
+ while len(result) != total:
+ if len(l) == i:
+ result += m[j:]
+ break
+ elif len(m) == j:
+ result += l[i:]
+ break
+ elif l[i] < m[j]:
+ result.append(l[i])
+ i += 1
+ else:
+ result.append(m[j])
+ j += 1
+ return result
+
+if __name__ == "__main__":
+ main()
diff --git a/applications/bed/sort-bed/src/update-sort-bed-slurm.py b/applications/bed/sort-bed/src/update-sort-bed-slurm.py
new file mode 100755
index 0000000..cb92e17
--- /dev/null
+++ b/applications/bed/sort-bed/src/update-sort-bed-slurm.py
@@ -0,0 +1,426 @@
+#!/usr/bin/env python
+
+#
+# BEDOPS
+# Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import sys
+import os
+import argparse
+import errno
+import subprocess
+import random
+import string
+import logging
+
+name = "update-sort-bed-slurm"
+citation = " citation: http://bioinformatics.oxfordjournals.org/content/28/14/1919.abstract"
+authors = " authors: Alex Reynolds and Shane Neph"
+version = " version: 2.4.26"
+usage = """ $ update-sort-bed-slurm [ --slurm-memory <MB> ]
+ [ --slurm-partition <SLURM partition> ]
+ [ --slurm-workdir <working directory> ]
+ [ --slurm-output <SLURM output directory> ]
+ [ --slurm-error <SLURM error directory> ]
+ [ --bedextract-path <path to bedextract> ]
+ [ --sort-bed-path <path to sort-bed> ]
+ [ --debug ]
+ --input-original <old-bed-file>
+ --input-backup <renamed-old-bed-file>
+ --output-temp <intermediate-new-bed-file>
+ --output-final <new-bed-file>"""
+help = """
+ The "update-sort-bed-slurm" utility applies an updated sort order on BED
+ files sorted per pre-v2.4.20 sort-bed, using a SLURM job scheduler to
+ coordinate resorting each chromosome in "--input-original" per post-v2.4.20
+ sort-bed and writing the result to "--output-final".
+
+ When migration is finished, "--input-backup" specifies the new name of the
+ original input.
+
+ As migration progresses, intermediate results are written to "--output-temp"
+ and then written to "--output-final" upon completion.
+
+ Each sort task is given 8 GB of memory and is assigned to the "queue0"
+ partition, unless the "--slurm-memory" and "--slurm-partition" options are
+ set. If your input is larger than 8 GB, you will need to allocate more
+ memory.
+
+ Because this launches all work on the specified cluster partition, the paths
+ specified by "--input-original", "--input-backup", "--output-final", and
+ "--output-temp" must be accessible to all computational nodes. For example,
+ using /tmp may fail, as the /tmp path is almost certainly unique to a node;
+ it is necesssary to use a path shared among all nodes.
+
+ Note that this utility will not work on entirely unsorted BED files, but only
+ on files with a sort order from pre-v2.4.20 sort-bed, where there are ties on
+ the first three columns.
+
+ In fact, until further refinements are made, this convenience utility could
+ fail silently on inputs which are not BED, or which are not sorted per pre-
+ v2.4.20 order, or which do not follow exact specification, all of which can
+ lead a per-chromosome resort task to fail.
+"""
+
+def main():
+ slurm_memory = "8000"
+ slurm_partition = "queue0"
+ slurm_workdir = os.getcwd()
+ slurm_output = None
+ slurm_error = None
+
+ # concatenation and other minor steps do not require much memory
+ slurm_backup_input_memory = "500"
+ slurm_concatenation_memory = "500"
+ slurm_cleanup_memory = "500"
+ slurm_output_move_memory = "500"
+
+ parser = argparse.ArgumentParser(prog=name, usage=usage, add_help=False)
+ parser.add_argument('--help', '-h', action='store_true', dest='help')
+ parser.add_argument('--slurm-memory', '-m', type=str, action="store", dest='slurm_memory')
+ parser.add_argument('--slurm-partition', '-p', type=str, action="store", dest='slurm_partition')
+ parser.add_argument('--slurm-workdir', '-w', type=str, action="store", dest='slurm_workdir')
+ parser.add_argument('--slurm-output', '-u', type=str, action="store", dest='slurm_output')
+ parser.add_argument('--slurm-error', '-e', type=str, action="store", dest='slurm_error')
+ parser.add_argument('--input-original', '-i', type=str, action="store", dest='input_original_fn')
+ parser.add_argument('--input-backup', '-b', type=str, action="store", dest='input_backup_fn')
+ parser.add_argument('--output-temp', '-t', type=str, action="store", dest='output_temp_fn')
+ parser.add_argument('--output-final', '-o', type=str, action="store", dest='output_final_fn')
+ parser.add_argument('--bedextract-path', '-x', type=str, action="store", dest='bedextract_path')
+ parser.add_argument('--sort-bed-path', '-s', type=str, action="store", dest='sort_bed_path')
+ parser.add_argument('--debug', '-d', action="store_true", dest='debug')
+ args = parser.parse_args()
+
+ if args.help or (not args.input_original_fn or not args.input_backup_fn or not args.output_temp_fn or not args.output_final_fn):
+ sys.stdout.write(name + '\n')
+ sys.stdout.write(citation + '\n')
+ sys.stdout.write(version + '\n')
+ sys.stdout.write(authors + '\n\n')
+ sys.stdout.write(usage + '\n')
+ sys.stdout.write(help)
+ if args.help:
+ sys.exit(os.EX_OK)
+ else:
+ sys.exit(errno.EINVAL)
+
+ logger = None
+ if args.debug:
+ logging.basicConfig(level=logging.DEBUG)
+ logger = logging.getLogger(__name__)
+
+ if logger: logger.info('Locating original input file [%s]' % (args.input_original_fn))
+ if not os.path.exists(args.input_original_fn):
+ if logger: logger.info('Not able to locate original input file')
+ sys.stderr.write("ERROR: Input file [%s] does not exist\n" % (args.input_original_fn))
+ sys.exit(errno.EINVAL)
+
+ if logger: logger.info('Locating final output file [%s]' % (args.output_final_fn))
+ if os.path.exists(args.output_final_fn):
+ if logger: logger.info('Found output file [%s] which will be replaced' % (args.output_final_fn))
+ sys.stderr.write("Note: Output file [%s] exists and will be replaced\n" % (args.output_final_fn))
+
+ if logger: logger.info('Verfying that \"sbatch\" can be found')
+ if not cmd_exists('sbatch'):
+ sys.stderr.write("ERROR: This script must be run on a system with SLURM binaries available\n")
+ sys.exit(errno.EEXIST)
+
+ if logger: logger.info('Testing if original input and backup input file paths are equal')
+ if args.input_original_fn == args.input_backup_fn:
+ sys.stderr.write("ERROR: Input filename [%s] cannot be the same as the input backup filename [%s]\n" % (args.input_original_fn, args.input_backup_fn))
+ sys.exit(errno.EINVAL)
+
+ if logger: logger.info('Testing if temporary output and final output file paths are equal')
+ if args.output_temp_fn == args.output_final_fn:
+ sys.stderr.write("ERROR: Output filename [%s] cannot be the same as the output temporary filename [%s]\n" % (args.output_final_fn, args.output_temp_fn))
+ sys.exit(errno.EINVAL)
+
+ # parse args
+ if args.slurm_memory:
+ slurm_memory = args.slurm_memory
+ if logger: logger.info('SLURM memory allocation set to [%s]' % (slurm_memory))
+
+ if args.slurm_partition:
+ slurm_partition = args.slurm_partition
+ if logger: logger.info('SLURM partition set to [%s]' % (slurm_partition))
+
+ if args.slurm_workdir:
+ slurm_workdir = args.slurm_workdir
+ if logger: logger.info('SLURM work directory set to [%s]' % (slurm_workdir))
+
+ if args.slurm_output:
+ slurm_output = args.slurm_output
+ if logger: logger.info('SLURM output log set to [%s]' % (slurm_output))
+
+ if args.slurm_error:
+ slurm_error = args.slurm_error
+ if logger: logger.info('SLURM error log set to [%s]' % (slurm_error))
+
+ if not slurm_output:
+ slurm_output = slurm_workdir
+ if logger: logger.info('SLURM output log reassigned to [%s]' % (slurm_output))
+
+ if not slurm_error:
+ slurm_error = slurm_workdir
+ if logger: logger.info('SLURM error log reassigned to [%s]' % (slurm_error))
+
+ bedextract_path = None
+ if logger: logger.info('Locating \"bedextract\" binary')
+ if not args.bedextract_path:
+ bedextract_path = find_binary('bedextract')
+ if not bedextract_path:
+ if logger: logger.info('Could not locate \"bedextract\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS bedextract\n")
+ sys.exit(errno.EEXIST)
+ elif not cmd_exists('bedextract'):
+ if logger: logger.info('Could not locate \"bedextract\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS bedextract\n")
+ sys.exit(errno.EEXIST)
+ else:
+ bedextract_path = args.bedextract_path
+ if logger: logger.info('Location of \"bedextract\" is set to [%s]' % (bedextract_path))
+
+ sort_bed_path = None
+ if logger: logger.info('Locating \"sort-bed\" binary')
+ if not args.sort_bed_path:
+ sort_bed_path = find_binary('sort-bed')
+ if not sort_bed_path:
+ if logger: logger.info('Could not locate \"sort-bed\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS sort-bed\n")
+ sys.exit(errno.EEXIST)
+ elif not cmd_exists('sort-bed'):
+ if logger: logger.info('Could not locate \"sort-bed\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS sort-bed\n")
+ sys.exit(errno.EEXIST)
+ else:
+ sort_bed_path = args.sort_bed_path
+ if logger: logger.info('Location of \"sort-bed\" is set to [%s]' % (sort_bed_path))
+
+ # build a list of chromosomes upon which to do work
+ list_chromosome_cmd_components = [
+ bedextract_path,
+ '--list-chr',
+ args.input_original_fn
+ ]
+ if logger: logger.info('Listing BED chromosomes via [%s]' % (' '.join(list_chromosome_cmd_components)))
+ try:
+ list_chromosome_cmd_result = subprocess.check_output(list_chromosome_cmd_components)
+ except subprocess.CalledProcessError as err:
+ list_chromosome_cmd_result = "ERROR: Command '{}' returned with error (code {}): {}".format(err.cmd, err.returncode, err.output)
+ raise
+ chromosome_list = list_chromosome_cmd_result.decode('utf-8').rstrip('\n').split('\n')
+ if logger: logger.info('Chromosomes to sort are [%s]' % (str(chromosome_list)))
+
+ # fire up the mid-range saloons^H^H^H per-chromosome sort tasks
+ job_prefix = ''.join(random.choice(string.lowercase) for x in range(8))
+ if logger: logger.info('Setting job prefix to [%s]' % (job_prefix))
+
+ per_chr_job_ids = []
+ per_chr_job_fns = []
+
+ for chromosome in chromosome_list:
+ temp_dest = os.path.join(os.getcwd(), '_'.join([job_prefix, chromosome]))
+ if logger: logger.info('Per-chromosome [%s] job name set to [%s]' % (chromosome, temp_dest))
+ per_chromosome_sort_cmd_components = [
+ 'sbatch',
+ '--parsable',
+ '--workdir',
+ slurm_workdir,
+ '--output',
+ os.path.join(slurm_output, '_'.join([job_prefix, 'out', chromosome])),
+ '--error',
+ os.path.join(slurm_error, '_'.join([job_prefix, 'err', chromosome])),
+ '--mem',
+ slurm_memory,
+ '--partition',
+ slurm_partition,
+ '--wrap',
+ '"srun ' + bedextract_path + ' ' + chromosome + ' ' + args.input_original_fn + ' | ' + sort_bed_path + ' - > ' + temp_dest + '"'
+ ]
+ if logger: logger.info('Submitting job via [%s]' % (' '.join(per_chromosome_sort_cmd_components)))
+ try:
+ per_chromosome_process = subprocess.Popen(' '.join(per_chromosome_sort_cmd_components),
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ close_fds=True)
+ (per_chromosome_stdout, per_chromosome_stderr) = per_chromosome_process.communicate()
+ per_chr_job_id = per_chromosome_stdout.decode('utf-8').rstrip('\n')
+ per_chr_job_ids.append(per_chr_job_id)
+ if logger: logger.info('Per-chromosome [%s] job id set to [%s]' % (chromosome, per_chr_job_id))
+ per_chr_job_fns.append(temp_dest)
+ except subprocess.CalledProcessError as err:
+ sys.stderr.write("ERROR: Per-chromosome sort task submission [%s] returned with error (code %d)\n" % (' '.join(err.cmd), err.returncode))
+ sys.exit(errno.EINVAL)
+ if logger: logger.info('Per-chromosome sort task outputs [%s] to be concatenated' % (str(per_chr_job_fns)))
+ per_chr_dependency_string = 'afterok:' + ':'.join(per_chr_job_ids)
+
+ # concatenate per-chromosome resorted files to output temp file
+ concatenation_cmd_components = [
+ 'sbatch',
+ '--parsable',
+ '--workdir',
+ slurm_workdir,
+ '--output',
+ os.path.join(slurm_output, '_'.join([job_prefix, 'out', 'concatenation'])),
+ '--error',
+ os.path.join(slurm_error, '_'.join([job_prefix, 'err', 'concatenation'])),
+ '--mem',
+ slurm_concatenation_memory,
+ '--partition',
+ slurm_partition,
+ '--dependency',
+ per_chr_dependency_string,
+ '--wrap',
+ '"srun cat ' + ' '.join(per_chr_job_fns) + ' > ' + args.output_temp_fn + '"'
+ ]
+ if logger: logger.info('Submitting concatenation job via [%s]' % (' '.join(concatenation_cmd_components)))
+ try:
+ concatenation_process = subprocess.Popen(' '.join(concatenation_cmd_components),
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ close_fds=True)
+ (concatenation_stdout, concatenation_stderr) = concatenation_process.communicate()
+ concatenation_job_id = concatenation_stdout.decode('utf-8').rstrip('\n')
+ if logger: logger.info('Concatenation job id set to [%s]' % (concatenation_job_id))
+ concatenation_job_dependency_string = 'afterok:' + concatenation_job_id
+ except subprocess.CalledProcessError as err:
+ sys.stderr.write("ERROR: Concatenation task submission [%s] returned with error (code %d)\n" % (' '.join(err.cmd), err.returncode))
+ sys.exit(errno.EINVAL)
+
+ # move original input file to backup file
+ backup_input_cmd_components = [
+ 'sbatch',
+ '--parsable',
+ '--workdir',
+ slurm_workdir,
+ '--output',
+ os.path.join(slurm_output, '_'.join([job_prefix, 'out', 'backup_input'])),
+ '--error',
+ os.path.join(slurm_error, '_'.join([job_prefix, 'err', 'backup_input'])),
+ '--mem',
+ slurm_backup_input_memory,
+ '--partition',
+ slurm_partition,
+ '--dependency',
+ concatenation_job_dependency_string,
+ '--wrap',
+ '"srun mv ' + args.input_original_fn + ' ' + args.input_backup_fn + '"'
+ ]
+ if logger: logger.info('Submitting backup job via [%s]' % (' '.join(backup_input_cmd_components)))
+ try:
+ backup_input_process = subprocess.Popen(' '.join(backup_input_cmd_components),
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ close_fds=True)
+ (backup_input_stdout, backup_input_stderr) = backup_input_process.communicate()
+ backup_input_job_id = backup_input_stdout.decode('utf-8').rstrip('\n')
+ if logger: logger.info('Backup job id set to [%s]' % (backup_input_job_id))
+ backup_input_job_dependency_string = 'afterok:' + backup_input_job_id
+ except subprocess.CalledProcessError as err:
+ sys.stderr.write("ERROR: Backup input task submission [%s] returned with error (code %d)\n" % (' '.join(err.cmd), err.returncode))
+ sys.exit(errno.EINVAL)
+
+ # move output temp file to output final file
+ output_move_cmd_components = [
+ 'sbatch',
+ '--parsable',
+ '--workdir',
+ slurm_workdir,
+ '--output',
+ os.path.join(slurm_output, '_'.join([job_prefix, 'out', 'concatenation'])),
+ '--error',
+ os.path.join(slurm_error, '_'.join([job_prefix, 'err', 'concatenation'])),
+ '--mem',
+ slurm_output_move_memory,
+ '--partition',
+ slurm_partition,
+ '--dependency',
+ backup_input_job_dependency_string,
+ '--wrap',
+ '"srun mv ' + args.output_temp_fn + ' ' + args.output_final_fn + '"'
+ ]
+ if logger: logger.info('Submitting final move job via [%s]' % (' '.join(output_move_cmd_components)))
+ try:
+ output_move_process = subprocess.Popen(' '.join(output_move_cmd_components),
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ close_fds=True)
+ (output_move_stdout, output_move_stderr) = output_move_process.communicate()
+ output_move_job_id = output_move_stdout.decode('utf-8').rstrip('\n')
+ if logger: logger.info('Final move job id set to [%s]' % (output_move_job_id))
+ output_move_job_dependency_string = 'afterok:' + output_move_job_id
+ except subprocess.CalledProcessError as err:
+ sys.stderr.write("ERROR: Output move task submission [%s] returned with error (code %d)\n" % (' '.join(err.cmd), err.returncode))
+ sys.exit(errno.EINVAL)
+
+ # perform cleanup
+ cleanup_cmd_components = [
+ 'sbatch',
+ '--parsable',
+ '--workdir',
+ slurm_workdir,
+ '--output',
+ os.path.join(slurm_output, '_'.join([job_prefix, 'out', 'concatenation'])),
+ '--error',
+ os.path.join(slurm_error, '_'.join([job_prefix, 'err', 'concatenation'])),
+ '--mem',
+ slurm_cleanup_memory,
+ '--partition',
+ slurm_partition,
+ '--dependency',
+ output_move_job_dependency_string,
+ '--wrap',
+ '"srun rm -f ' + os.path.join(slurm_workdir, job_prefix) + '_* ' + os.path.join(slurm_output, job_prefix) + '_*"'
+ ]
+ if logger: logger.info('Submitting cleanup job via [%s]' % (' '.join(cleanup_cmd_components)))
+ try:
+ cleanup_process = subprocess.Popen(' '.join(cleanup_cmd_components),
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ close_fds=True)
+ (cleanup_stdout, cleanup_stderr) = cleanup_process.communicate()
+ cleanup_job_id = cleanup_stdout.decode('utf-8').rstrip('\n')
+ if logger: logger.info('Cleanup job id set to [%s]' % (cleanup_job_id))
+ except subprocess.CalledProcessError as err:
+ sys.stderr.write("ERROR: Cleanup task submission [%s] returned with error (code %d)\n" % (' '.join(err.cmd), err.returncode))
+ sys.exit(errno.EINVAL)
+
+def find_binary(binary_to_find):
+ for p in os.environ['PATH'].split(':'):
+ for r, d, f in os.walk(p):
+ for filename in f:
+ if filename == binary_to_find:
+ return os.path.join(r, filename)
+ return None
+
+def cmd_exists(cmd):
+ return subprocess.call("type " + cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0
+
+if __name__ == "__main__":
+ main()
diff --git a/applications/bed/sort-bed/src/update-sort-bed-starch-slurm.py b/applications/bed/sort-bed/src/update-sort-bed-starch-slurm.py
new file mode 100755
index 0000000..038a858
--- /dev/null
+++ b/applications/bed/sort-bed/src/update-sort-bed-starch-slurm.py
@@ -0,0 +1,462 @@
+#!/usr/bin/env python
+
+#
+# BEDOPS
+# Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import sys
+import os
+import argparse
+import errno
+import subprocess
+import random
+import string
+import logging
+
+name = "update-sort-bed-starch-slurm"
+citation = " citation: http://bioinformatics.oxfordjournals.org/content/28/14/1919.abstract"
+authors = " authors: Alex Reynolds and Shane Neph"
+version = " version: 2.4.26"
+usage = """ $ update-sort-bed-starch-slurm [ --slurm-memory <MB> ]
+ [ --slurm-partition <SLURM partition> ]
+ [ --slurm-workdir <working directory> ]
+ [ --slurm-output <SLURM output directory> ]
+ [ --slurm-error <SLURM error directory> ]
+ [ --bedextract-path <path to bedextract> ]
+ [ --sort-bed-path <path to sort-bed> ]
+ [ --unstarch-path <path to unstarch> ]
+ [ --starch-path <path to starch> ]
+ [ --starchcat-path <path to starchcat> ]
+ [ --debug ]
+ --input-original <old-bed-file>
+ --input-backup <renamed-old-bed-file>
+ --output-temp <intermediate-new-bed-file>
+ --output-final <new-bed-file>"""
+help = """
+ The 'update-sort-bed-starch-slurm' utility applies an updated sort order on
+ Starch files sorted per pre-v2.4.20 sort-bed, using a SLURM job scheduler to
+ coordinate resorting each chromosome in "--input-original" per post-v2.4.20
+ sort-bed and writing the result to a Starch file at "--output-final".
+
+ When migration is finished, "--input-backup" specifies the new name of the
+ original input file.
+
+ As migration progresses, intermediate results are written to "--output-temp"
+ and then written to "--output-final" upon completion.
+
+ Each sort task is given 8 GB of memory and is assigned to the "queue0"
+ partition, unless the "--slurm-memory" and "--slurm-partition" options are
+ set. If your input is larger than 8 GB, you will need to allocate more
+ memory.
+
+ Because this launches all work on the specified cluster partition, the paths
+ specified by "--input-original", "--input-backup", "--output-final", and
+ "--output-temp" must be accessible to all computational nodes. For example,
+ using /tmp may fail, as the /tmp path is almost certainly unique to a node;
+ it is necesssary to use a path shared among all nodes.
+
+ Note that this utility will not work on entirely unsorted BED files, but only
+ on files with a sort order from pre-v2.4.20 sort-bed, where there are ties on
+ the first three columns.
+
+ In fact, until further refinements are made, this convenience utility could
+ fail silently on inputs which are not BED, or which are not sorted per pre-
+ v2.4.20 order, or which do not follow exact specification, all of which can
+ lead a per-chromosome resort task to fail.
+"""
+
+def main():
+ slurm_memory = "8000"
+ slurm_partition = "queue0"
+ slurm_workdir = os.getcwd()
+ slurm_output = None
+ slurm_error = None
+
+ # concatenation and other minor steps do not require much memory
+ slurm_backup_input_memory = "500"
+ slurm_concatenation_memory = "500"
+ slurm_cleanup_memory = "500"
+ slurm_output_move_memory = "500"
+
+ parser = argparse.ArgumentParser(prog=name, usage=usage, add_help=False)
+ parser.add_argument('--help', '-h', action='store_true', dest='help')
+ parser.add_argument('--slurm-memory', '-m', type=str, action="store", dest='slurm_memory')
+ parser.add_argument('--slurm-partition', '-p', type=str, action="store", dest='slurm_partition')
+ parser.add_argument('--slurm-workdir', '-w', type=str, action="store", dest='slurm_workdir')
+ parser.add_argument('--slurm-output', '-u', type=str, action="store", dest='slurm_output')
+ parser.add_argument('--slurm-error', '-e', type=str, action="store", dest='slurm_error')
+ parser.add_argument('--input-original', '-i', type=str, action="store", dest='input_original_fn')
+ parser.add_argument('--input-backup', '-b', type=str, action="store", dest='input_backup_fn')
+ parser.add_argument('--output-temp', '-t', type=str, action="store", dest='output_temp_fn')
+ parser.add_argument('--output-final', '-o', type=str, action="store", dest='output_final_fn')
+ parser.add_argument('--sort-bed-path', '-s', type=str, action="store", dest='sort_bed_path')
+ parser.add_argument('--unstarch-path', '-a', type=str, action="store", dest='unstarch_path')
+ parser.add_argument('--starch-path', '-c', type=str, action="store", dest='starch_path')
+ parser.add_argument('--starchcat-path', '-z', type=str, action="store", dest='starchcat_path')
+ parser.add_argument('--debug', '-d', action="store_true", dest='debug')
+ args = parser.parse_args()
+
+ if args.help or (not args.input_original_fn or not args.input_backup_fn or not args.output_temp_fn or not args.output_final_fn):
+ sys.stdout.write(name + '\n')
+ sys.stdout.write(citation + '\n')
+ sys.stdout.write(version + '\n')
+ sys.stdout.write(authors + '\n\n')
+ sys.stdout.write(usage + '\n')
+ sys.stdout.write(help)
+ if args.help:
+ sys.exit(os.EX_OK)
+ else:
+ sys.exit(errno.EINVAL)
+
+ logger = None
+ if args.debug:
+ logging.basicConfig(level=logging.DEBUG)
+ logger = logging.getLogger(__name__)
+
+ if logger: logger.info('Locating original input file [%s]' % (args.input_original_fn))
+ if not os.path.exists(args.input_original_fn):
+ if logger: logger.info('Not able to locate original input file')
+ sys.stderr.write("ERROR: Input file [%s] does not exist\n" % (args.input_original_fn))
+ sys.exit(errno.EINVAL)
+
+ if logger: logger.info('Locating final output file [%s]' % (args.output_final_fn))
+ if os.path.exists(args.output_final_fn):
+ if logger: logger.info('Found output file [%s] which will be replaced' % (args.output_final_fn))
+ sys.stderr.write("Note: Output file [%s] exists and will be replaced\n" % (args.output_final_fn))
+
+ if logger: logger.info('Verfying that \"sbatch\" can be found')
+ if not cmd_exists('sbatch'):
+ sys.stderr.write("ERROR: This script must be run on a system with SLURM binaries available\n")
+ sys.exit(errno.EEXIST)
+
+ if logger: logger.info('Testing if original input and backup input file paths are equal')
+ if args.input_original_fn == args.input_backup_fn:
+ sys.stderr.write("ERROR: Input filename [%s] cannot be the same as the input backup filename [%s]\n" % (args.input_original_fn, args.input_backup_fn))
+ sys.exit(errno.EINVAL)
+
+ if logger: logger.info('Testing if temporary output and final output file paths are equal')
+ if args.output_temp_fn == args.output_final_fn:
+ sys.stderr.write("ERROR: Output filename [%s] cannot be the same as the output temporary filename [%s]\n" % (args.output_final_fn, args.output_temp_fn))
+ sys.exit(errno.EINVAL)
+
+ # parse args
+ if args.slurm_memory:
+ slurm_memory = args.slurm_memory
+ if logger: logger.info('SLURM memory allocation set to [%s]' % (slurm_memory))
+
+ if args.slurm_partition:
+ slurm_partition = args.slurm_partition
+ if logger: logger.info('SLURM partition set to [%s]' % (slurm_partition))
+
+ if args.slurm_workdir:
+ slurm_workdir = args.slurm_workdir
+ if logger: logger.info('SLURM work directory set to [%s]' % (slurm_workdir))
+
+ if args.slurm_output:
+ slurm_output = args.slurm_output
+ if logger: logger.info('SLURM output log set to [%s]' % (slurm_output))
+
+ if args.slurm_error:
+ slurm_error = args.slurm_error
+ if logger: logger.info('SLURM error log set to [%s]' % (slurm_error))
+
+ if not slurm_output:
+ slurm_output = slurm_workdir
+ if logger: logger.info('SLURM output log reassigned to [%s]' % (slurm_output))
+
+ if not slurm_error:
+ slurm_error = slurm_workdir
+ if logger: logger.info('SLURM error log reassigned to [%s]' % (slurm_error))
+
+ sort_bed_path = None
+ if logger: logger.info('Locating \"sort-bed\" binary')
+ if not args.sort_bed_path:
+ sort_bed_path = find_binary('sort-bed')
+ if not sort_bed_path:
+ if logger: logger.info('Could not locate \"sort-bed\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS sort-bed\n")
+ sys.exit(errno.EEXIST)
+ elif not cmd_exists('sort-bed'):
+ if logger: logger.info('Could not locate \"sort-bed\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS sort-bed\n")
+ sys.exit(errno.EEXIST)
+ else:
+ sort_bed_path = args.sort_bed_path
+ if logger: logger.info('Location of \"sort-bed\" is set to [%s]' % (sort_bed_path))
+
+ unstarch_path = None
+ if logger: logger.info('Locating \"unstarch\" binary')
+ if not args.unstarch_path:
+ unstarch_path = find_binary('unstarch')
+ if not unstarch_path:
+ if logger: logger.info('Could not locate \"unstarch\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS unstarch\n")
+ sys.exit(errno.EEXIST)
+ elif not cmd_exists('unstarch'):
+ if logger: logger.info('Could not locate \"unstarch\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS unstarch\n")
+ sys.exit(errno.EEXIST)
+ else:
+ unstarch_path = args.unstarch_path
+ if logger: logger.info('Location of \"unstarch\" is set to [%s]' % (unstarch_path))
+
+ starch_path = None
+ if logger: logger.info('Locating \"starch\" binary')
+ if not args.starch_path:
+ starch_path = find_binary('starch')
+ if not starch_path:
+ if logger: logger.info('Could not locate \"starch\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS starch\n")
+ sys.exit(errno.EEXIST)
+ elif not cmd_exists('starch'):
+ if logger: logger.info('Could not locate \"starch\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS starch\n")
+ sys.exit(errno.EEXIST)
+ else:
+ starch_path = args.starch_path
+ if logger: logger.info('Location of \"starch\" is set to [%s]' % (starch_path))
+
+ starchcat_path = None
+ if logger: logger.info('Locating \"starchcat\" binary')
+ if not args.starchcat_path:
+ starchcat_path = find_binary('starchcat')
+ if not starchcat_path:
+ if logger: logger.info('Could not locate \"starchcat\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS starchcat\n")
+ sys.exit(errno.EEXIST)
+ elif not cmd_exists('starchcat'):
+ if logger: logger.info('Could not locate \"starchcat\" binary')
+ sys.stderr.write("ERROR: This script must be run on a system with BEDOPS starchcat\n")
+ sys.exit(errno.EEXIST)
+ else:
+ starchcat_path = args.starchcat_path
+ if logger: logger.info('Location of \"starchcat\" is set to [%s]' % (starchcat_path))
+
+ # build a list of chromosomes upon which to do work
+ list_chromosome_cmd_components = [
+ unstarch_path,
+ '--list-chr',
+ args.input_original_fn
+ ]
+ if logger: logger.info('Listing Starch chromosomes via [%s]' % (' '.join(list_chromosome_cmd_components)))
+ try:
+ list_chromosome_cmd_result = subprocess.check_output(list_chromosome_cmd_components)
+ except subprocess.CalledProcessError as err:
+ list_chromosome_cmd_result = "ERROR: Command '{}' returned with error (code {}): {}".format(err.cmd, err.returncode, err.output)
+ raise
+ chromosome_list = list_chromosome_cmd_result.decode('utf-8').rstrip('\n').split('\n')
+ if logger: logger.info('Chromosomes to sort are [%s]' % (str(chromosome_list)))
+
+ # fire up the mid-range saloons^H^H^H per-chromosome sort tasks
+ job_prefix = ''.join(random.choice(string.lowercase) for x in range(8))
+ if logger: logger.info('Setting job prefix to [%s]' % (job_prefix))
+
+ per_chr_job_ids = []
+ per_chr_job_fns = []
+
+ for chromosome in chromosome_list:
+ temp_dest = os.path.join(os.getcwd(), '_'.join([job_prefix, chromosome]))
+ if logger: logger.info('Per-chromosome [%s] job name set to [%s]' % (chromosome, temp_dest))
+ per_chromosome_sort_cmd_components = [
+ 'sbatch',
+ '--parsable',
+ '--workdir',
+ slurm_workdir,
+ '--output',
+ os.path.join(slurm_output, '_'.join([job_prefix, 'out', chromosome])),
+ '--error',
+ os.path.join(slurm_error, '_'.join([job_prefix, 'err', chromosome])),
+ '--mem',
+ slurm_memory,
+ '--partition',
+ slurm_partition,
+ '--wrap',
+ '"srun ' + unstarch_path + ' ' + chromosome + ' ' + args.input_fn + ' | ' + sort_bed_path + ' - | ' + starch_path + ' - > ' + temp_dest + '"'
+ ]
+ if logger: logger.info('Submitting job via [%s]' % (' '.join(per_chromosome_sort_cmd_components)))
+ try:
+ per_chromosome_process = subprocess.Popen(' '.join(per_chromosome_sort_cmd_components),
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ close_fds=True)
+ (per_chromosome_stdout, per_chromosome_stderr) = per_chromosome_process.communicate()
+ per_chr_job_id = per_chromosome_stdout.decode('utf-8').rstrip('\n')
+ per_chr_job_ids.append(per_chr_job_id)
+ if logger: logger.info('Per-chromosome [%s] job id set to [%s]' % (chromosome, per_chr_job_id))
+ per_chr_job_fns.append(temp_dest)
+ except subprocess.CalledProcessError as err:
+ sys.stderr.write("ERROR: Per-chromosome sort task submission [%s] returned with error (code %d)\n" % (' '.join(err.cmd), err.returncode))
+ sys.exit(errno.EINVAL)
+ if logger: logger.info('Per-chromosome sort task outputs [%s] to be concatenated' % (str(per_chr_job_fns)))
+ per_chr_dependency_string = 'afterok:' + ':'.join(per_chr_job_ids)
+
+ # concatenate per-chromosome resorted files to output temp file
+ concatenation_cmd_components = [
+ 'sbatch',
+ '--parsable',
+ '--workdir',
+ slurm_workdir,
+ '--output',
+ os.path.join(slurm_output, '_'.join([job_prefix, 'out', 'concatenation'])),
+ '--error',
+ os.path.join(slurm_error, '_'.join([job_prefix, 'err', 'concatenation'])),
+ '--mem',
+ slurm_concatenation_memory,
+ '--partition',
+ slurm_partition,
+ '--dependency',
+ per_chr_dependency_string,
+ '--wrap',
+ '"srun ' + starchcat_path + ' ' + ' '.join(per_chr_job_fns) + ' > ' + args.output_temp_fn + '"'
+ ]
+ if logger: logger.info('Submitting concatenation job via [%s]' % (' '.join(concatenation_cmd_components)))
+ try:
+ concatenation_process = subprocess.Popen(' '.join(concatenation_cmd_components),
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ close_fds=True)
+ (concatenation_stdout, concatenation_stderr) = concatenation_process.communicate()
+ concatenation_job_id = concatenation_stdout.decode('utf-8').rstrip('\n')
+ if logger: logger.info('Concatenation job id set to [%s]' % (concatenation_job_id))
+ concatenation_job_dependency_string = 'afterok:' + concatenation_job_id
+ except subprocess.CalledProcessError as err:
+ sys.stderr.write("ERROR: Concatenation task submission [%s] returned with error (code %d)\n" % (' '.join(err.cmd), err.returncode))
+ sys.exit(errno.EINVAL)
+
+ # move original input file to backup file
+ backup_input_cmd_components = [
+ 'sbatch',
+ '--parsable',
+ '--workdir',
+ slurm_workdir,
+ '--output',
+ os.path.join(slurm_output, '_'.join([job_prefix, 'out', 'backup_input'])),
+ '--error',
+ os.path.join(slurm_error, '_'.join([job_prefix, 'err', 'backup_input'])),
+ '--mem',
+ slurm_backup_input_memory,
+ '--partition',
+ slurm_partition,
+ '--dependency',
+ concatenation_job_dependency_string,
+ '--wrap',
+ '"srun mv ' + args.input_original_fn + ' ' + args.input_backup_fn + '"'
+ ]
+ if logger: logger.info('Submitting backup job via [%s]' % (' '.join(backup_input_cmd_components)))
+ try:
+ backup_input_process = subprocess.Popen(' '.join(backup_input_cmd_components),
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ close_fds=True)
+ (backup_input_stdout, backup_input_stderr) = backup_input_process.communicate()
+ backup_input_job_id = backup_input_stdout.decode('utf-8').rstrip('\n')
+ if logger: logger.info('Backup job id set to [%s]' % (backup_input_job_id))
+ backup_input_job_dependency_string = 'afterok:' + backup_input_job_id
+ except subprocess.CalledProcessError as err:
+ sys.stderr.write("ERROR: Backup input task submission [%s] returned with error (code %d)\n" % (' '.join(err.cmd), err.returncode))
+ sys.exit(errno.EINVAL)
+
+ # move output temp file to output final file
+ output_move_cmd_components = [
+ 'sbatch',
+ '--parsable',
+ '--workdir',
+ slurm_workdir,
+ '--output',
+ os.path.join(slurm_output, '_'.join([job_prefix, 'out', 'concatenation'])),
+ '--error',
+ os.path.join(slurm_error, '_'.join([job_prefix, 'err', 'concatenation'])),
+ '--mem',
+ slurm_output_move_memory,
+ '--partition',
+ slurm_partition,
+ '--dependency',
+ backup_input_job_dependency_string,
+ '--wrap',
+ '"srun mv ' + args.output_temp_fn + ' ' + args.output_final_fn + '"'
+ ]
+ if logger: logger.info('Submitting final move job via [%s]' % (' '.join(output_move_cmd_components)))
+ try:
+ output_move_process = subprocess.Popen(' '.join(output_move_cmd_components),
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ close_fds=True)
+ (output_move_stdout, output_move_stderr) = output_move_process.communicate()
+ output_move_job_id = output_move_stdout.decode('utf-8').rstrip('\n')
+ if logger: logger.info('Final move job id set to [%s]' % (output_move_job_id))
+ output_move_job_dependency_string = 'afterok:' + output_move_job_id
+ except subprocess.CalledProcessError as err:
+ sys.stderr.write("ERROR: Output move task submission [%s] returned with error (code %d)\n" % (' '.join(err.cmd), err.returncode))
+ sys.exit(errno.EINVAL)
+
+ # perform cleanup
+ cleanup_cmd_components = [
+ 'sbatch',
+ '--parsable',
+ '--workdir',
+ slurm_workdir,
+ '--output',
+ os.path.join(slurm_output, '_'.join([job_prefix, 'out', 'concatenation'])),
+ '--error',
+ os.path.join(slurm_error, '_'.join([job_prefix, 'err', 'concatenation'])),
+ '--mem',
+ slurm_cleanup_memory,
+ '--partition',
+ slurm_partition,
+ '--dependency',
+ output_move_job_dependency_string,
+ '--wrap',
+ '"srun rm -f ' + os.path.join(slurm_workdir, job_prefix) + '_* ' + os.path.join(slurm_output, job_prefix) + '_*"'
+ ]
+ try:
+ cleanup_process = subprocess.Popen(' '.join(cleanup_cmd_components),
+ shell=True,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ close_fds=True)
+ (cleanup_stdout, cleanup_stderr) = cleanup_process.communicate()
+ cleanup_job_id = cleanup_stdout.decode('utf-8').rstrip('\n')
+ if logger: logger.info('Cleanup job id set to [%s]' % (cleanup_job_id))
+ except subprocess.CalledProcessError as err:
+ sys.stderr.write("ERROR: Cleanup task submission [%s] returned with error (code %d)\n" % (' '.join(err.cmd), err.returncode))
+ sys.exit(errno.EINVAL)
+
+def find_binary(binary_to_find):
+ for p in os.environ['PATH'].split(':'):
+ for r, d, f in os.walk(p):
+ for filename in f:
+ if filename == binary_to_find:
+ return os.path.join(r, filename)
+ return None
+
+def cmd_exists(cmd):
+ return subprocess.call("type " + cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0
+
+if __name__ == "__main__":
+ main()
diff --git a/applications/bed/starch/src/Makefile b/applications/bed/starch/src/Makefile
index 7865386..fc81548 100644
--- a/applications/bed/starch/src/Makefile
+++ b/applications/bed/starch/src/Makefile
@@ -45,11 +45,11 @@ CXXGFLAGS = -D__STDC_CONSTANT_MACROS -D_FILE_OFFSET_BITS=64 -D_L
STARCH_NAMES = starchConstants unstarchHelpers starchHelpers starchMetadataHelpers starchFileHelpers starchSha1Digest starchBase64Coding
STARCH_OBJECTS = $(addprefix $(LOCALOBJDIR)/, $(addsuffix .o, $(STARCH_NAMES)))
-.PHONY: starchcluster starch unstarch starchcat build build_debug build_gprof
+.PHONY: starchcluster starch unstarch starchcat build build_debug build_gprof starch-diff starchstrip
-build: starch unstarch starchcat starchcluster
-build_debug: $(BINDIR)/debug.starch $(BINDIR)/debug.unstarch $(BINDIR)/debug.starchcat
-build_gprof: $(BINDIR)/gprof.starch $(BINDIR)/gprof.unstarch $(BINDIR)/gprof.starchcat
+build: starch unstarch starchcat starchcluster starch-diff starchstrip
+build_debug: $(BINDIR)/debug.starch $(BINDIR)/debug.unstarch $(BINDIR)/debug.starchcat $(BINDIR)/debug.starchstrip
+build_gprof: $(BINDIR)/gprof.starch $(BINDIR)/gprof.unstarch $(BINDIR)/gprof.starchcat $(BINDIR)/gprof.starchstrip
# Cancel implicit compilation rules
% : %.o
@@ -58,7 +58,9 @@ build_gprof: $(BINDIR)/gprof.starch $(BINDIR)/gprof.unstarch $(BINDIR)/gprof.sta
starch: $(BINDIR)/starch
unstarch: $(BINDIR)/unstarch
starchcat: $(BINDIR)/starchcat
-starchcluster: $(BINDIR)/starchcluster_gnuParallel $(BINDIR)/starchcluster_sge
+starchcluster: $(BINDIR)/starchcluster_gnuParallel $(BINDIR)/starchcluster_sge $(BINDIR)/starchcluster_slurm
+starch-diff: $(BINDIR)/starch-diff
+starchstrip: $(BINDIR)/starchstrip
$(BINDIR)/% : %.c $(LOCALSTARCHLIB) $(LIBRARIES)
mkdir -p $(BINDIR)
@@ -79,6 +81,10 @@ $(BINDIR)/% : %.tcsh
mkdir -p $(BINDIR)
cp $< $@
+$(BINDIR)/% : %.py
+ mkdir -p $(BINDIR)
+ cp $< $@
+
starchLibrary: $(LOCALSTARCHLIB)
$(LOCALSTARCHLIB) : $(STARCH_OBJECTS)
diff --git a/applications/bed/starch/src/Makefile.darwin b/applications/bed/starch/src/Makefile.darwin
index 664086d..eac99a8 100644
--- a/applications/bed/starch/src/Makefile.darwin
+++ b/applications/bed/starch/src/Makefile.darwin
@@ -32,18 +32,17 @@ INCLUDES = -iquote${MAIN} -iquote${HEAD} -iquote${PARTY3} -I${L
LIBRARIES = ${LOCALJANSSONLIB} ${LOCALBZIP2LIB} ${LOCALZLIBLIB}
BINDIR = ../bin_${ARCH}
WARNINGS = -Weverything -Wno-c++98-compat-pedantic -Wno-padded
-ARCH_VERSION = v2.1
-BIN_VERSION = v2.4.16
+ARCH_VERSION = v2.2
+BIN_VERSION = v2.4.26
TEST = ../test
TEST_OSX_BINDIR = ${TEST}/binaries/osx/${ARCH_VERSION}/bin
CFLAGS = -D__STDC_CONSTANT_MACROS -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE=1 -DUSE_ZLIB -DUSE_BZLIB -O2 ${WARNINGS} -std=c99 -mmacosx-version-min=$(MIN_OSX_VERSION) -arch ${ARCH} -x c++ -v -stdlib=libc++ -std=c++11
CXXFLAGS = -D__STDC_CONSTANT_MACROS -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE=1 -DUSE_ZLIB -DUSE_BZLIB -O2 ${WARNINGS} -std=c++11 -stdlib=libc++ -mmacosx-version-min=$(MIN_OSX_VERSION) -arch ${ARCH}
-CDFLAGS = -D__STDC_CONSTANT_MACROS -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE=1 -DUSE_ZLIB -DUSE_BZLIB -O0 -g ${WARNINGS} -std=c99 -DDEBUG_VERBOSE=1 -mmacosx-version-min=$(MIN_OSX_VERSION) -arch ${ARCH} -x c++ -v -stdlib=libc++ -std=c++11
-CXXDFLAGS = -D__STDC_CONSTANT_MACROS -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE=1 -DUSE_ZLIB -DUSE_BZLIB -O0 -g ${WARNINGS} -std=c++11 -stdlib=libc++ -DDEBUG_VERBOSE=1 -mmacosx-version-min=$(MIN_OSX_VERSION) -arch ${ARCH}
+CDFLAGS = -D__STDC_CONSTANT_MACROS -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE=1 -DUSE_ZLIB -DUSE_BZLIB -O0 -g ${WARNINGS} -std=c99 -DDEBUG=1 -mmacosx-version-min=$(MIN_OSX_VERSION) -arch ${ARCH} -x c++ -v -stdlib=libc++ -std=c++11
+CXXDFLAGS = -D__STDC_CONSTANT_MACROS -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE=1 -DUSE_ZLIB -DUSE_BZLIB -O0 -g ${WARNINGS} -std=c++11 -stdlib=libc++ -DDEBUG=1 -mmacosx-version-min=$(MIN_OSX_VERSION) -arch ${ARCH}
-
-build: dependencies starchLibrary starch unstarch starchcat starchcluster
+build: dependencies starchLibrary starch unstarch starchcat starchcluster starch-diff starchstrip
rm -f *~
# dependencies-related recipes all have mkdirs prereqs
@@ -58,7 +57,7 @@ dependencies: mkdirs
${CC} ${CFLAGS} -c ${OBJDIR}/starchSha1Digest.c -o ${LOCALOBJDIR}/starchSha1Digest.o ${INCLUDES}
${CC} ${CFLAGS} -c ${OBJDIR}/starchBase64Coding.c -o ${LOCALOBJDIR}/starchBase64Coding.o ${INCLUDES}
-build_debug: dependencies_debug starch_debug unstarch_debug starchcat_debug
+build_debug: dependencies_debug starchLibrary_debug starch_debug unstarch_debug starchcat_debug starchstrip_debug
rm -f *~
dependencies_debug: mkdirs
@@ -95,6 +94,10 @@ unstarch_debug: starchLibrary_debug
starchcluster: starchcat
cp starchcluster_sge.tcsh ${BINDIR}/starchcluster_sge
cp starchcluster_gnuParallel.tcsh ${BINDIR}/starchcluster_gnuParallel
+ cp starchcluster_slurm.tcsh ${BINDIR}/starchcluster_slurm
+
+starch-diff: unstarch
+ cp starch-diff.py ${BINDIR}/starch-diff
starchcat: starchLibrary
${CC} ${CFLAGS} -c starchcat.c -o $(LOCALOBJDIR)/starchcat.o ${INCLUDES}
@@ -104,6 +107,14 @@ starchcat_debug: starchLibrary_debug
${CC} ${CDFLAGS} -c starchcat.c -o $(LOCALOBJDIR)/starchcat.o ${INCLUDES}
${CXX} ${CXXDFLAGS} -lc++ $(LOCALOBJDIR)/starchcat.o -o ${BINDIR}/debug.starchcat ${LOCALSTARCHLIBDEBUG} ${LIBRARIES}
+starchstrip: starchLibrary
+ ${CC} ${CFLAGS} -c starchstrip.c -o $(LOCALOBJDIR)/starchstrip.o ${INCLUDES}
+ ${CXX} ${CXXFLAGS} -lc++ $(LOCALOBJDIR)/starchstrip.o -o ${BINDIR}/starchstrip ${LOCALSTARCHLIB} ${LIBRARIES}
+
+starchstrip_debug: starchLibrary_debug
+ ${CC} ${CDFLAGS} -c starchstrip.c -o $(LOCALOBJDIR)/debug.starchstrip.o ${INCLUDES}
+ ${CXX} ${CXXDFLAGS} -lc++ $(LOCALOBJDIR)/debug.starchstrip.o -o ${BINDIR}/debug.starchstrip ${LOCALSTARCHLIBDEBUG} ${LIBRARIES}
+
test: starch unstarch starchcat
cp ${BINDIR}/starch ${TEST_OSX_BINDIR}/starch
cp ${BINDIR}/unstarch ${TEST_OSX_BINDIR}/unstarch
diff --git a/applications/bed/starch/src/starch-diff.py b/applications/bed/starch/src/starch-diff.py
new file mode 100755
index 0000000..e989c1f
--- /dev/null
+++ b/applications/bed/starch/src/starch-diff.py
@@ -0,0 +1,213 @@
+#!/usr/bin/env python
+
+#
+# BEDOPS
+# Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import sys
+import os
+import argparse
+import errno
+import subprocess
+import json
+import logging
+
+name = "starch-diff"
+citation = " citation: http://bioinformatics.oxfordjournals.org/content/28/14/1919.abstract"
+authors = " authors: Alex Reynolds and Shane Neph"
+version = " version: 2.4.26"
+usage = " $ starch-diff [ --chr <chr> ] starch-file-1 starch-file-2 [ starch-file-3 ... ]"
+help = """
+ The 'starch-diff' utility compares the signatures of two or more specified
+ Starch v2.2+ archives for all chromosomes, or for a specified chromosome.
+"""
+
+default_chromosome = "all"
+
+def main():
+ parser = argparse.ArgumentParser(prog=name, usage=usage, add_help=False)
+ parser.add_argument('--help', '-h', action='store_true', dest='help')
+ parser.add_argument('--chr', '-c', type=str, action="store", default=default_chromosome)
+ parser.add_argument('--debug', '-d', action='store_true', dest='debug')
+ parser.add_argument('file', type=argparse.FileType('r'), nargs='*')
+ args = parser.parse_args()
+
+ if args.debug:
+ logging.basicConfig(level=logging.DEBUG)
+ logger = logging.getLogger(__name__)
+
+ if args.help or len(args.file) < 2:
+ sys.stdout.write(name + '\n')
+ sys.stdout.write(citation + '\n')
+ sys.stdout.write(version + '\n')
+ sys.stdout.write(authors + '\n\n')
+ sys.stdout.write(usage + '\n')
+ sys.stdout.write(help)
+ if args.help:
+ sys.exit(os.EX_OK)
+ else:
+ sys.stdout.write("\nERROR: Please specify two or more Starch archives as input\n")
+ sys.exit(errno.EINVAL)
+
+ all_tests_pass = True
+
+ selected_chromosome = unicode(args.chr)
+ if args.debug: logger.info('Selected chromosome [%s]' % (selected_chromosome))
+
+ archive_paths = list()
+ for i in range(0, len(args.file)):
+ archive_handle = args.file[i]
+ if not os.path.exists(archive_handle.name) or not os.path.isfile(archive_handle.name):
+ sys.stdout.write("ERROR: Input [%s] is not a file or does not exist\n" % (archive_handle.name))
+ sys.stdout.write("%s\n" % (usage))
+ sys.exit(errno.ENOENT)
+ archive_paths.append(archive_handle.name)
+ if args.debug: logger.info('Appended input file to list [%s]' % (archive_handle.name))
+ num_files = len(args.file)
+ if args.debug: logger.info('Number of input files [%d]' % (num_files))
+
+ if selected_chromosome == default_chromosome:
+ if args.debug: logger.info('Examining all chromosomes from all inputs...')
+ all_chromosomes = {}
+ for archive_path in archive_paths:
+ get_archive_version_cmd_components = [
+ 'unstarch',
+ '--list-json',
+ archive_path
+ ]
+ try:
+ get_archive_version_cmd_result = subprocess.check_output(get_archive_version_cmd_components)
+ except subprocess.CalledProcessError as err:
+ get_archive_version_cmd_result = "ERROR: Command '{}' returned with error (code {}): {}".format(err.cmd, err.returncode, err.output)
+ raise
+ archive_metadata = json.loads(get_archive_version_cmd_result.decode('utf-8'))
+ try:
+ archive_version = archive_metadata['archive']['version']
+ except KeyError as err:
+ sys.stderr.write("ERROR: Could not read archive version from Starch metadata\n")
+ raise
+ if archive_version['major'] < 2 or (archive_version['major'] == 2 and archive_version['minor'] < 2):
+ sys.stderr.write("ERROR: Input [%s] must be a v2.2+ Starch archive -- use 'starchcat' or extract/recompress to update archive\n" % (archive_path))
+ sys.exit(errno.EINVAL)
+
+ list_chromosome_cmd_components = [
+ 'unstarch',
+ '--list-chr',
+ archive_path
+ ]
+ try:
+ list_chromosome_cmd_result = subprocess.check_output(list_chromosome_cmd_components)
+ except subprocess.CalledProcessError as err:
+ list_chromosome_cmd_result = "ERROR: Command '{}' returned with error (code {}): {}".format(err.cmd, err.returncode, err.output)
+ raise
+ archive_chromosomes = list_chromosome_cmd_result.rstrip('\n').split('\n')
+ for chromosome in archive_chromosomes:
+ if chromosome not in all_chromosomes:
+ all_chromosomes[chromosome] = 0
+ all_chromosomes[chromosome] += 1
+
+ common_chromosomes = [k for k, v in all_chromosomes.iteritems() if v == num_files]
+ common_chromosomes.sort()
+ if args.debug: logger.info('Common chromosomes [%s]' % (','.join(common_chromosomes)))
+
+ uncommon_chromosomes = [k for k, v in all_chromosomes.iteritems() if v != num_files]
+ uncommon_chromosomes.sort()
+ if args.debug: logger.info('Uncommon chromosomes [%s]' % (','.join(uncommon_chromosomes)))
+
+ if not common_chromosomes:
+ sys.stderr.write("ERROR: Inputs share no records with a common chromosome name\n")
+ sys.exit(errno.EINVAL)
+ else:
+ common_chromosomes = [selected_chromosome]
+
+ tests_to_run = {}
+ for common_chromosome in common_chromosomes:
+ if common_chromosome not in tests_to_run:
+ tests_to_run[common_chromosome] = []
+
+ for idx, archive_path in enumerate(archive_paths):
+ get_chromosome_signature_cmd_components = [
+ 'unstarch',
+ common_chromosome,
+ '--signature',
+ archive_path
+ ]
+ try:
+ get_chromosome_signature_cmd_result = subprocess.check_output(get_chromosome_signature_cmd_components)
+ except subprocess.CalledProcessError as err:
+ sys.stderr.write("ERROR: Command [%s] returned with error (code %d)\n" % (' '.join(err.cmd), err.returncode))
+ sys.exit(errno.EINVAL)
+
+ chromosome_signature = get_chromosome_signature_cmd_result.rstrip('\n')
+ if len(chromosome_signature) == 0:
+ chromosome_signature = None
+
+ result = {}
+ result['archive'] = archive_path
+ result['signature'] = chromosome_signature
+ tests_to_run[common_chromosome].append(result)
+ if args.debug: logger.info('Test to run: [%s] [%s]' % (common_chromosome, str(result)))
+
+ # for a given chromosome, we have a set of objects to test for equality
+ for test_chromosome in sorted(tests_to_run.keys()):
+ per_chromosome_tests_pass = True
+ previous_signature = None
+ current_signature = None
+ previous_archive = None
+ current_archive = None
+ none_signature_found = False
+ test_list = tests_to_run[test_chromosome]
+ for test_item in test_list:
+ test_signature = test_item['signature']
+ test_archive = test_item['archive']
+ if args.debug: logger.info('Examining archive [%s]' % (test_archive))
+ if args.debug: logger.info('Setting current signature [%s] to [%s]' % (current_signature, test_signature))
+ current_signature = test_signature
+ current_archive = test_archive
+ if not test_signature:
+ none_signature_found = True
+ per_chromosome_tests_pass = False
+ break
+ elif not previous_signature:
+ if args.debug: logger.info('Setting previous signature [%s] to [%s]' % (previous_signature, test_signature))
+ previous_signature = test_signature
+ previous_archive = test_archive
+ if args.debug: logger.info('Comparing chr [%s] previous signature [%s] current signature [%s]' % (test_chromosome, previous_signature, current_signature))
+ if current_signature != previous_signature:
+ per_chromosome_tests_pass = False
+ if not per_chromosome_tests_pass:
+ all_tests_pass = False
+ if none_signature_found:
+ sys.stderr.write('WARNING: One or more signatures are not available for chromosome [%s] in archive [%s]\n' % (test_chromosome, current_archive))
+ else:
+ sys.stderr.write('WARNING: Signatures do not match for chromosome [%s] between archives [%s] and [%s]\n' % (test_chromosome, previous_archive, current_archive))
+
+ if not all_tests_pass:
+ sys.exit(errno.EINVAL)
+
+ if selected_chromosome == default_chromosome:
+ sys.stderr.write('Compressed genomic streams in input files are identical\n')
+ else:
+ sys.stderr.write('Compressed genomic streams in input files are identical for chromosome [%s]\n' % (selected_chromosome))
+
+ sys.exit(os.EX_OK)
+
+if __name__ == "__main__":
+ main()
diff --git a/applications/bed/starch/src/starch.c b/applications/bed/starch/src/starch.c
index ea6fc3b..3e793ab 100644
--- a/applications/bed/starch/src/starch.c
+++ b/applications/bed/starch/src/starch.c
@@ -6,7 +6,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -53,6 +53,9 @@ main (int argc, char **argv)
char *bedFn = NULL;
FILE *bedFnPtr = NULL;
Metadata *metadata = NULL;
+ Boolean bedGeneratePerChrSignatureFlag = kStarchFalse;
+ Boolean bedReportProgressFlag = kStarchFalse;
+ LineCountType bedReportProgressN = 0;
Boolean bedHeaderFlag = kStarchFalse;
unsigned char *starchHeader = NULL;
@@ -77,6 +80,9 @@ main (int argc, char **argv)
bedFn = starch_client_global_args.inputFile;
type = starch_client_global_args.compressionType;
tag = starch_client_global_args.uniqueTag;
+ bedGeneratePerChrSignatureFlag = starch_client_global_args.generatePerChromosomeSignatureFlag;
+ bedReportProgressFlag = starch_client_global_args.reportProgressFlag;
+ bedReportProgressN = starch_client_global_args.reportProgressN;
bedHeaderFlag = starch_client_global_args.headerFlag;
if (STARCH_MAJOR_VERSION == 1)
@@ -86,13 +92,13 @@ main (int argc, char **argv)
/* process stdin */
if ((bedHeaderFlag == kStarchTrue) &&
#ifdef __cplusplus
- (STARCH_transformInput(&metadata,
+ (STARCH_transformInput(&metadata,
NULL,
static_cast<const CompressionType>( type ),
reinterpret_cast<const char *>( tag ),
reinterpret_cast<const char *>( note )) != 0))
#else
- (STARCH_transformInput(&metadata,
+ (STARCH_transformInput(&metadata,
NULL,
(const CompressionType) type,
(const char *) tag,
@@ -165,29 +171,35 @@ main (int argc, char **argv)
bedFnPtr = STARCH_fopen (bedFn, "r");
if (!bedFnPtr) {
fprintf (stderr, "ERROR: Could not open input BED file pointer from (%s)\n", bedFn);
- exit (EXIT_FAILURE);
- }
- }
+ exit (EXIT_FAILURE);
+ }
+ }
#ifdef __cplusplus
if (STARCH2_transformInput(&starchHeader,
&metadata,
- reinterpret_cast<const FILE *>( bedFnPtr ),
- static_cast<const CompressionType>( type ),
- reinterpret_cast<const char *>( tag ),
- reinterpret_cast<const char *>( note ),
- static_cast<const Boolean>( bedHeaderFlag )) != STARCH_EXIT_SUCCESS)
+ reinterpret_cast<const FILE *>( bedFnPtr ),
+ static_cast<const CompressionType>( type ),
+ reinterpret_cast<const char *>( tag ),
+ reinterpret_cast<const char *>( note ),
+ static_cast<const Boolean>( bedGeneratePerChrSignatureFlag ),
+ static_cast<const Boolean>( bedHeaderFlag ),
+ static_cast<const Boolean>( bedReportProgressFlag ),
+ static_cast<const LineCountType>( bedReportProgressN )) != STARCH_EXIT_SUCCESS)
{
exit (EXIT_FAILURE);
}
#else
if (STARCH2_transformInput(&starchHeader,
&metadata,
- (const FILE *) bedFnPtr,
- (const CompressionType) type,
- (const char *) tag,
- (const char *) note,
- (const Boolean) bedHeaderFlag) != STARCH_EXIT_SUCCESS)
+ (const FILE *) bedFnPtr,
+ (const CompressionType) type,
+ (const char *) tag,
+ (const char *) note,
+ (const Boolean) bedGeneratePerChrSignatureFlag,
+ (const Boolean) bedHeaderFlag,
+ (const Boolean) bedReportProgressFlag,
+ (const LineCountType) bedReportProgressN) != STARCH_EXIT_SUCCESS)
{
exit (EXIT_FAILURE);
}
@@ -228,6 +240,9 @@ STARCH_initializeGlobals ()
#endif
starch_client_global_args.note = NULL;
starch_client_global_args.compressionType = STARCH_DEFAULT_COMPRESSION_TYPE;
+ starch_client_global_args.generatePerChromosomeSignatureFlag = kStarchTrue;
+ starch_client_global_args.reportProgressFlag = kStarchFalse;
+ starch_client_global_args.reportProgressN = 0;
starch_client_global_args.headerFlag = kStarchFalse;
starch_client_global_args.inputFile = NULL;
starch_client_global_args.uniqueTag = NULL;
@@ -250,32 +265,45 @@ STARCH_parseCommandLineOptions (int argc, char **argv)
return STARCH_FATAL_ERROR;
}
- opterr = 0; /* disable error reporting by GNU getopt -- we handle this */
+ /* disable error reporting by GNU getopt -- we handle this */
+ opterr = 0;
STARCH_initializeGlobals ();
while (starch_client_opt != -1) {
switch (starch_client_opt) {
- case 'v':
- return STARCH_VERSION_ERROR;
- case 'n':
- starch_client_global_args.note = optarg;
- break;
- case 'b':
- starch_client_global_args.compressionType = kBzip2;
- break;
- case 'g':
- starch_client_global_args.compressionType = kGzip;
- break;
- case 'e':
- starch_client_global_args.headerFlag = kStarchTrue;
- break;
- case 'h':
- return STARCH_HELP_ERROR;
- case '?':
+ case 'v':
+ return STARCH_VERSION_ERROR;
+ case 'n':
+ starch_client_global_args.note = optarg;
+ break;
+ case 'b':
+ starch_client_global_args.compressionType = kBzip2;
+ break;
+ case 'g':
+ starch_client_global_args.compressionType = kGzip;
+ break;
+ case 'o':
+ starch_client_global_args.generatePerChromosomeSignatureFlag = kStarchFalse;
+ break;
+ case 'r':
+ starch_client_global_args.reportProgressFlag = kStarchTrue;
+ errno = 0;
+ starch_client_global_args.reportProgressN = strtoumax(optarg, NULL, 10);
+ if (errno == ERANGE) {
+ fprintf (stderr, "ERROR: Numerical value is outside of range.\n");
return STARCH_FATAL_ERROR;
- default:
- break;
- }
+ }
+ break;
+ case 'e':
+ starch_client_global_args.headerFlag = kStarchTrue;
+ break;
+ case 'h':
+ return STARCH_HELP_ERROR;
+ case '?':
+ return STARCH_FATAL_ERROR;
+ default:
+ break;
+ }
starch_client_opt = getopt_long (argc, argv, starch_client_opt_string, starch_client_long_options, &starch_client_long_index);
}
@@ -356,7 +384,7 @@ STARCH_printRevision ()
if (avStr != NULL) {
int result = sprintf (avStr, "%d.%d.%d", STARCH_MAJOR_VERSION, STARCH_MINOR_VERSION, STARCH_REVISION_VERSION);
if (result != -1)
- fprintf (stderr, "%s\n binary version: %s (creates archive version: %s)\n", name, BEDOPS::revision(), avStr);
+ fprintf (stderr, "%s\n binary version: %s (creates archive version: %s)\n", name, BEDOPS::revision(), avStr);
free (avStr);
}
}
diff --git a/applications/bed/starch/src/starch.h b/applications/bed/starch/src/starch.h
index 3321f19..79e213f 100644
--- a/applications/bed/starch/src/starch.h
+++ b/applications/bed/starch/src/starch.h
@@ -6,7 +6,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -27,6 +27,8 @@
#define STARCH_H
#include <getopt.h>
+#include <inttypes.h>
+#include <errno.h>
#include "data/starch/starchMetadataHelpers.h"
@@ -39,24 +41,40 @@ namespace {
static const char *name = "starch";
static const char *authors = "Alex Reynolds and Shane Neph";
static const char *usage = "\n" \
- "USAGE: starch [--note=\"foo bar...\"] [--bzip2 | --gzip] [--header] [<unique-tag>] <bed-file>\n" \
+ "USAGE: starch [ --note=\"foo bar...\" ]\n" \
+ " [ --bzip2 | --gzip ]\n" \
+ " [ --omit-signature ]\n" \
+ " [ --report-progress=N ]\n" \
+ " [ --header ] [ <unique-tag> ] <bed-file>\n" \
" \n" \
" * BED input must be sorted lexicographically (e.g., using BEDOPS sort-bed).\n" \
" * Please use '-' to indicate reading BED data from standard input.\n" \
" * Output must be directed to a regular file.\n" \
- " * The bzip2 compression type makes smaller archives, while gzip extracts faster.\n" \
+ " * The bzip2 compression type makes smaller archives, while gzip extracts\n" \
+ " faster.\n" \
" \n" \
- " Process Flags:\n\n" \
- " --note=\"foo bar...\" Append note to output archive metadata (optional)\n" \
- " --bzip2 | --gzip Specify backend compression type (optional, default is bzip2)\n" \
- " --header Support BED input with custom UCSC track, SAM or VCF headers, or generic comments (optional)\n" \
- " <unique-tag> Specify unique identifier for transformed data (optional)\n" \
- " --help Show this usage message\n" \
- " --version Show binary version";
+ " Process Flags\n" \
+ " --------------------------------------------------------------------------\n" \
+ " --note=\"foo bar...\" Append note to output archive metadata (optional).\n\n" \
+ " --bzip2 | --gzip Specify backend compression type (optional, default\n" \
+ " is bzip2).\n\n" \
+ " --omit-signature Skip generating per-chromosome data integrity signature\n" \
+ " (optional, default is to generate signature).\n\n" \
+ " --report-progress=N Report compression progress every N elements per\n" \
+ " chromosome to standard error stream (optional)\n\n" \
+ " --header Support BED input with custom UCSC track, SAM or VCF\n" \
+ " headers, or generic comments (optional).\n\n" \
+ " <unique-tag> Optional. Specify unique identifier for transformed\n" \
+ " data.\n\n" \
+ " --version Show binary version.\n\n" \
+ " --help Show this usage message.\n";
static struct starch_client_global_args_t {
char *note;
CompressionType compressionType;
+ Boolean generatePerChromosomeSignatureFlag;
+ Boolean reportProgressFlag;
+ LineCountType reportProgressN;
Boolean headerFlag;
char *inputFile;
char *uniqueTag;
@@ -66,25 +84,18 @@ static struct starch_client_global_args_t {
} starch_client_global_args;
static struct option starch_client_long_options[] = {
- {"note", required_argument, NULL, 'n'},
- {"bzip2", no_argument, NULL, 'b'},
- {"gzip", no_argument, NULL, 'g'},
- {"header", no_argument, NULL, 'e'},
- {"version", no_argument, NULL, 'v'},
- {"help", no_argument, NULL, 'h'},
- {NULL, no_argument, NULL, 0}
+ {"note", required_argument, NULL, 'n'},
+ {"bzip2", no_argument, NULL, 'b'},
+ {"gzip", no_argument, NULL, 'g'},
+ {"omit-signature", no_argument, NULL, 'o'},
+ {"report-progress", required_argument, NULL, 'r'},
+ {"header", no_argument, NULL, 'e'},
+ {"version", no_argument, NULL, 'v'},
+ {"help", no_argument, NULL, 'h'},
+ {NULL, no_argument, NULL, 0 }
};
-static const char *starch_client_opt_string = "n:bgevh?";
-
-/*
- On Darwin, file I/O is 64-bit by default (OS X 10.5 at least) so we use standard
- types and calls
-*/
-
-#ifdef __APPLE__
-#define off64_t off_t
-#endif
+static const char *starch_client_opt_string = "n:bgorevh?";
#ifdef __cplusplus
namespace starch {
diff --git a/applications/bed/starch/src/starchcat.c b/applications/bed/starch/src/starchcat.c
index e54e749..4f86292 100644
--- a/applications/bed/starch/src/starchcat.c
+++ b/applications/bed/starch/src/starchcat.c
@@ -6,7 +6,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -82,6 +82,9 @@ main (int argc, char **argv)
json_t **metadataJSONs = NULL;
size_t cumulativeRecSize = 0U;
unsigned char *header = NULL;
+ Boolean bedGeneratePerChrSignatureFlag = kStarchFalse;
+ Boolean bedReportProgressFlag = kStarchFalse;
+ LineCountType bedReportProgressN = 0;
setlocale(LC_ALL, "POSIX");
@@ -92,6 +95,9 @@ main (int argc, char **argv)
parseResult = STARCHCAT_parseCommandLineOptions(argc, argv);
note = starchcat_client_global_args.note;
outputType = starchcat_client_global_args.compressionType;
+ bedGeneratePerChrSignatureFlag = starchcat_client_global_args.generatePerChromosomeSignatureFlag;
+ bedReportProgressFlag = starchcat_client_global_args.reportProgressFlag;
+ bedReportProgressN = starchcat_client_global_args.reportProgressN;
#ifdef __cplusplus
firstArgc = static_cast<unsigned int>( argc ) - static_cast<unsigned int>( starchcat_client_global_args.numberInputFiles );
#else
@@ -145,12 +151,12 @@ main (int argc, char **argv)
if (STARCH_MAJOR_VERSION == 1) {
#ifdef __cplusplus
assert( STARCHCAT_mergeChromosomeStreams ( reinterpret_cast<const ChromosomeSummaries *>( summaries ),
- static_cast<const CompressionType>( outputType ),
- reinterpret_cast<const char *>( note ) ) );
+ static_cast<const CompressionType>( outputType ),
+ reinterpret_cast<const char *>( note ) ) );
#else
assert( STARCHCAT_mergeChromosomeStreams ( (const ChromosomeSummaries *) summaries,
- (const CompressionType) outputType,
- (const char *) note ) );
+ (const CompressionType) outputType,
+ (const char *) note ) );
#endif
}
else if (STARCH_MAJOR_VERSION == 2) {
@@ -159,14 +165,20 @@ main (int argc, char **argv)
cumulativeRecSize += STARCH2_MD_HEADER_BYTE_LENGTH;
#ifdef __cplusplus
assert( STARCHCAT2_mergeChromosomeStreams ( reinterpret_cast<const ChromosomeSummaries *>( summaries ),
- static_cast<const CompressionType>( outputType ),
- reinterpret_cast<const char *>( note ),
- &cumulativeRecSize ) );
+ static_cast<const CompressionType>( outputType ),
+ reinterpret_cast<const char *>( note ),
+ &cumulativeRecSize,
+ bedGeneratePerChrSignatureFlag,
+ bedReportProgressFlag,
+ bedReportProgressN ) );
#else
assert( STARCHCAT2_mergeChromosomeStreams ( (const ChromosomeSummaries *) summaries,
- (const CompressionType) outputType,
- (const char *) note,
- &cumulativeRecSize ) );
+ (const CompressionType) outputType,
+ (const char *) note,
+ &cumulativeRecSize,
+ bedGeneratePerChrSignatureFlag,
+ bedReportProgressFlag,
+ bedReportProgressN ) );
#endif
}
break;
@@ -235,6 +247,7 @@ STARCHCAT_initializeGlobals()
starchcat_client_global_args.compressionType = STARCH_DEFAULT_COMPRESSION_TYPE;
starchcat_client_global_args.numberInputFiles = 0;
starchcat_client_global_args.inputFiles = NULL;
+ starchcat_client_global_args.generatePerChromosomeSignatureFlag = kStarchTrue;
}
int
@@ -262,6 +275,18 @@ STARCHCAT_parseCommandLineOptions(int argc, char **argv)
case 'g':
starchcat_client_global_args.compressionType = kGzip;
break;
+ case 'o':
+ starchcat_client_global_args.generatePerChromosomeSignatureFlag = kStarchFalse;
+ break;
+ case 'r':
+ starchcat_client_global_args.reportProgressFlag = kStarchTrue;
+ errno = 0;
+ starchcat_client_global_args.reportProgressN = strtoumax(optarg, NULL, 10);
+ if (errno == ERANGE) {
+ fprintf (stderr, "ERROR: Numerical value is outside of range.\n");
+ return STARCHCAT_FATAL_ERROR;
+ }
+ break;
case 'h':
return STARCHCAT_HELP_ERROR;
case '?':
@@ -284,7 +309,7 @@ STARCHCAT_parseCommandLineOptions(int argc, char **argv)
}
int
-STARCHCAT2_copyInputRecordToOutput (Metadata **outMd, const char *outTag, const CompressionType outType, const char *inChr, const MetadataRecord *inRec, size_t *cumulativeOutputSize)
+STARCHCAT2_copyInputRecordToOutput (Metadata **outMd, const char *outTag, const CompressionType outType, const char *inChr, const MetadataRecord *inRec, size_t *cumulativeOutputSize, const Boolean reportProgressFlag)
{
#ifdef DEBUG
fprintf (stderr, "\n--- STARCHCAT2_copyInputRecordToOutput() ---\n");
@@ -300,7 +325,9 @@ STARCHCAT2_copyInputRecordToOutput (Metadata **outMd, const char *outTag, const
uint64_t endOffset = 0;
uint64_t outFileSize = 0;
uint64_t outFileSizeCounter = 0;
+ char *outSignature = NULL;
LineCountType outFileLineCount = 0;
+ LineLengthType outFileLineMaxStringLength = STARCH_DEFAULT_LINE_STRING_LENGTH;
BaseCountType outFileNonUniqueBases = 0;
BaseCountType outFileUniqueBases = 0;
Boolean outDuplicateElementExists = STARCH_DEFAULT_DUPLICATE_ELEMENT_FLAG_VALUE;
@@ -320,7 +347,7 @@ STARCHCAT2_copyInputRecordToOutput (Metadata **outMd, const char *outTag, const
#ifdef __cplusplus
outFn = static_cast<char *>( malloc(strlen(inChr) + strlen(outTag) + 6) ); /* X.Y.bz2\0 */
#else
- outFn = malloc(strlen(inChr) + strlen(outTag) + 6); /* X.Y.bz2\0 */
+ outFn = malloc(strlen(inChr) + strlen(outTag) + 6); /* X.Y.bz2\0 */
#endif
if (!outFn) {
fprintf(stderr, "ERROR: Could not allocate space for output filename in input copy routine.\n");
@@ -366,10 +393,14 @@ STARCHCAT2_copyInputRecordToOutput (Metadata **outMd, const char *outTag, const
if (strcmp(iter->chromosome, inChr) == 0) {
if (((av->major == 1) && (av->minor >= 4)) || (av->major == 2)) {
outFileLineCount = iter->lineCount;
+ outFileLineMaxStringLength = iter->lineMaxStringLength;
outFileNonUniqueBases = iter->totalNonUniqueBases;
outFileUniqueBases = iter->totalUniqueBases;
outDuplicateElementExists = iter->duplicateElementExists;
outNestedElementExists = iter->nestedElementExists;
+ if (iter->signature) {
+ outSignature = STARCH_strndup(iter->signature, strlen(iter->signature) + 1);
+ }
}
else if ((av->major == 1) && (av->minor >= 3))
outFileLineCount = iter->lineCount;
@@ -423,40 +454,44 @@ STARCHCAT2_copyInputRecordToOutput (Metadata **outMd, const char *outTag, const
do {
if (outFileSizeCounter > STARCHCAT_COPY_BUFFER_MAXSIZE) {
#ifdef __cplusplus
- nBytesRead = fread(buffer, sizeof(char), static_cast<size_t>( STARCHCAT_COPY_BUFFER_MAXSIZE ), inRec->fp);
+ nBytesRead = fread(buffer, sizeof(char), static_cast<size_t>( STARCHCAT_COPY_BUFFER_MAXSIZE ), inRec->fp);
#else
- nBytesRead = fread(buffer, sizeof(char), (size_t) STARCHCAT_COPY_BUFFER_MAXSIZE, inRec->fp);
+ nBytesRead = fread(buffer, sizeof(char), (size_t) STARCHCAT_COPY_BUFFER_MAXSIZE, inRec->fp);
#endif
- if (nBytesRead != STARCHCAT_COPY_BUFFER_MAXSIZE * sizeof(char)) {
- fprintf(stderr, "ERROR: Was not able to copy sufficient bytes into buffer (STARCHCAT_COPY_BUFFER_MAXSIZE).\n");
- return STARCHCAT_EXIT_FAILURE;
- }
+ if (nBytesRead != STARCHCAT_COPY_BUFFER_MAXSIZE * sizeof(char)) {
+ fprintf(stderr, "ERROR: Was not able to copy sufficient bytes into buffer (STARCHCAT_COPY_BUFFER_MAXSIZE).\n");
+ return STARCHCAT_EXIT_FAILURE;
+ }
#ifdef __cplusplus
- fwrite(buffer, sizeof(char), static_cast<size_t>( STARCHCAT_COPY_BUFFER_MAXSIZE ), outFnPtr);
+ fwrite(buffer, sizeof(char), static_cast<size_t>( STARCHCAT_COPY_BUFFER_MAXSIZE ), outFnPtr);
#else
- fwrite(buffer, sizeof(char), (size_t) STARCHCAT_COPY_BUFFER_MAXSIZE, outFnPtr);
+ fwrite(buffer, sizeof(char), (size_t) STARCHCAT_COPY_BUFFER_MAXSIZE, outFnPtr);
#endif
- outFileSizeCounter -= STARCHCAT_COPY_BUFFER_MAXSIZE;
- }
- else {
+ outFileSizeCounter -= STARCHCAT_COPY_BUFFER_MAXSIZE;
+ }
+ else {
#ifdef __cplusplus
- nBytesRead = fread(buffer, sizeof(char), static_cast<size_t>( outFileSizeCounter ), inRec->fp);
+ nBytesRead = fread(buffer, sizeof(char), static_cast<size_t>( outFileSizeCounter ), inRec->fp);
#else
- nBytesRead = fread(buffer, sizeof(char), (size_t) outFileSizeCounter, inRec->fp);
+ nBytesRead = fread(buffer, sizeof(char), (size_t) outFileSizeCounter, inRec->fp);
#endif
- if (nBytesRead != outFileSizeCounter * sizeof(char)) {
- fprintf(stderr, "ERROR: Was not able to copy sufficient bytes into buffer (outFileSizeCounter).\n");
- return STARCHCAT_EXIT_FAILURE;
- }
+ if (nBytesRead != outFileSizeCounter * sizeof(char)) {
+ fprintf(stderr, "ERROR: Was not able to copy sufficient bytes into buffer (outFileSizeCounter).\n");
+ return STARCHCAT_EXIT_FAILURE;
+ }
#ifdef __cplusplus
- fwrite(buffer, sizeof(char), static_cast<size_t>( outFileSizeCounter ), outFnPtr);
+ fwrite(buffer, sizeof(char), static_cast<size_t>( outFileSizeCounter ), outFnPtr);
#else
- fwrite(buffer, sizeof(char), (size_t) outFileSizeCounter, outFnPtr);
+ fwrite(buffer, sizeof(char), (size_t) outFileSizeCounter, outFnPtr);
#endif
- outFileSizeCounter = 0ULL;
- }
+ outFileSizeCounter = 0ULL;
+ }
} while (outFileSizeCounter > 0);
+ if (reportProgressFlag) {
+ fprintf(stderr, "PROGRESS: Copied chromosome [%s] to output stream\n", inChr);
+ }
+
/* update output metadata */
if (! *outMd) {
#ifdef DEBUG
@@ -473,7 +508,9 @@ STARCHCAT2_copyInputRecordToOutput (Metadata **outMd, const char *outTag, const
outFileNonUniqueBases,
outFileUniqueBases,
outDuplicateElementExists,
- outNestedElementExists );
+ outNestedElementExists,
+ outSignature,
+ outFileLineMaxStringLength );
}
else {
#ifdef DEBUG
@@ -491,7 +528,9 @@ STARCHCAT2_copyInputRecordToOutput (Metadata **outMd, const char *outTag, const
outFileNonUniqueBases,
outFileUniqueBases,
outDuplicateElementExists,
- outNestedElementExists );
+ outNestedElementExists,
+ outSignature,
+ outFileLineMaxStringLength );
}
return STARCHCAT_EXIT_SUCCESS;
@@ -514,6 +553,7 @@ STARCHCAT_copyInputRecordToOutput (Metadata **outMd, const char *outTag, const C
uint64_t outFileSize = 0;
uint64_t outFileSizeCounter = 0;
LineCountType outFileLineCount = 0;
+ LineLengthType outFileLineMaxStringLength = STARCH_DEFAULT_LINE_STRING_LENGTH;
BaseCountType outFileNonUniqueBases = 0;
BaseCountType outFileUniqueBases = 0;
Boolean outDuplicateElementExists = STARCH_DEFAULT_DUPLICATE_ELEMENT_FLAG_VALUE;
@@ -523,6 +563,7 @@ STARCHCAT_copyInputRecordToOutput (Metadata **outMd, const char *outTag, const C
char buffer[STARCHCAT_COPY_BUFFER_MAXSIZE];
FILE *outFnPtr = NULL;
char *outFn = NULL;
+ char *outSignature = NULL;
size_t nBytesRead = 0;
if (!inMd) {
@@ -623,40 +664,40 @@ STARCHCAT_copyInputRecordToOutput (Metadata **outMd, const char *outTag, const C
fseeko(inRec->fp, (off_t) startOffset, SEEK_SET);
#endif
do {
- if (outFileSizeCounter > STARCHCAT_COPY_BUFFER_MAXSIZE) {
+ if (outFileSizeCounter > STARCHCAT_COPY_BUFFER_MAXSIZE) {
#ifdef __cplusplus
- nBytesRead = fread(buffer, sizeof(char), static_cast<size_t>( STARCHCAT_COPY_BUFFER_MAXSIZE ), inRec->fp);
+ nBytesRead = fread(buffer, sizeof(char), static_cast<size_t>( STARCHCAT_COPY_BUFFER_MAXSIZE ), inRec->fp);
#else
- nBytesRead = fread(buffer, sizeof(char), (size_t) STARCHCAT_COPY_BUFFER_MAXSIZE, inRec->fp);
+ nBytesRead = fread(buffer, sizeof(char), (size_t) STARCHCAT_COPY_BUFFER_MAXSIZE, inRec->fp);
#endif
- if (nBytesRead != STARCHCAT_COPY_BUFFER_MAXSIZE * sizeof(char)) {
- fprintf(stderr, "ERROR: Was not able to copy sufficient bytes into buffer (STARCHCAT_COPY_BUFFER_MAXSIZE).\n");
- return STARCHCAT_EXIT_FAILURE;
- }
+ if (nBytesRead != STARCHCAT_COPY_BUFFER_MAXSIZE * sizeof(char)) {
+ fprintf(stderr, "ERROR: Was not able to copy sufficient bytes into buffer (STARCHCAT_COPY_BUFFER_MAXSIZE).\n");
+ return STARCHCAT_EXIT_FAILURE;
+ }
#ifdef __cplusplus
- fwrite(buffer, sizeof(char), static_cast<size_t>( STARCHCAT_COPY_BUFFER_MAXSIZE ), outFnPtr);
+ fwrite(buffer, sizeof(char), static_cast<size_t>( STARCHCAT_COPY_BUFFER_MAXSIZE ), outFnPtr);
#else
- fwrite(buffer, sizeof(char), (size_t) STARCHCAT_COPY_BUFFER_MAXSIZE, outFnPtr);
+ fwrite(buffer, sizeof(char), (size_t) STARCHCAT_COPY_BUFFER_MAXSIZE, outFnPtr);
#endif
- outFileSizeCounter -= STARCHCAT_COPY_BUFFER_MAXSIZE;
- }
- else {
+ outFileSizeCounter -= STARCHCAT_COPY_BUFFER_MAXSIZE;
+ }
+ else {
#ifdef __cplusplus
- nBytesRead = fread(buffer, sizeof(char), static_cast<size_t>( outFileSizeCounter ), inRec->fp);
+ nBytesRead = fread(buffer, sizeof(char), static_cast<size_t>( outFileSizeCounter ), inRec->fp);
#else
- nBytesRead = fread(buffer, sizeof(char), (size_t) outFileSizeCounter, inRec->fp);
+ nBytesRead = fread(buffer, sizeof(char), (size_t) outFileSizeCounter, inRec->fp);
#endif
- if (nBytesRead != outFileSizeCounter * sizeof(char)) {
- fprintf(stderr, "ERROR: Was not able to copy sufficient bytes into buffer (outFileSizeCounter).\n");
- return STARCHCAT_EXIT_FAILURE;
- }
+ if (nBytesRead != outFileSizeCounter * sizeof(char)) {
+ fprintf(stderr, "ERROR: Was not able to copy sufficient bytes into buffer (outFileSizeCounter).\n");
+ return STARCHCAT_EXIT_FAILURE;
+ }
#ifdef __cplusplus
- fwrite(buffer, sizeof(char), static_cast<size_t>( outFileSizeCounter ), outFnPtr);
+ fwrite(buffer, sizeof(char), static_cast<size_t>( outFileSizeCounter ), outFnPtr);
#else
- fwrite(buffer, sizeof(char), (size_t) outFileSizeCounter, outFnPtr);
+ fwrite(buffer, sizeof(char), (size_t) outFileSizeCounter, outFnPtr);
#endif
- outFileSizeCounter = 0ULL;
- }
+ outFileSizeCounter = 0ULL;
+ }
} while (outFileSizeCounter > 0);
/* update output metadata */
@@ -675,7 +716,9 @@ STARCHCAT_copyInputRecordToOutput (Metadata **outMd, const char *outTag, const C
outFileNonUniqueBases,
outFileUniqueBases,
outDuplicateElementExists,
- outNestedElementExists);
+ outNestedElementExists,
+ outSignature,
+ outFileLineMaxStringLength);
}
else {
#ifdef DEBUG
@@ -693,7 +736,9 @@ STARCHCAT_copyInputRecordToOutput (Metadata **outMd, const char *outTag, const C
outFileNonUniqueBases,
outFileUniqueBases,
outDuplicateElementExists,
- outNestedElementExists);
+ outNestedElementExists,
+ outSignature,
+ outFileLineMaxStringLength);
}
/* fprintf(stderr, "\t\tchr: %s, outFn: %s, size: %llu\n", (*outMd)->chromosome, (*outMd)->filename, (*outMd)->size); */
@@ -704,7 +749,7 @@ STARCHCAT_copyInputRecordToOutput (Metadata **outMd, const char *outTag, const C
}
int
-STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, const CompressionType outType, const char *inChr, const MetadataRecord *inRec, size_t *cumulativeOutputSize)
+STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, const CompressionType outType, const char *inChr, const MetadataRecord *inRec, size_t *cumulativeOutputSize, const Boolean generatePerChrSignatureFlag, const Boolean reportProgressFlag, const LineCountType reportProgressN)
{
/*
This function extracts a single record (chromosome) of data
@@ -713,7 +758,6 @@ STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, con
stream compression type. The records metadata are updated at the end
with the statistics for the new record.
*/
-
#ifdef DEBUG
fprintf (stderr, "\n--- STARCHCAT2_rewriteInputRecordToOutput() ---\n");
#endif
@@ -725,42 +769,98 @@ STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, con
Metadata *iter, *inMd = inRec->metadata;
ArchiveVersion *av = inRec->av;
- /* intermediate buffer variaables */
- unsigned char retransformLineBuf[TOKENS_MAX_LENGTH] = {0};
+ /* intermediate buffer variables */
+ unsigned char *retransformLineBuf = NULL;
int64_t nRetransformLineBuf = 0;
int64_t nRetransformLineBufPos = 0;
- unsigned char retransformBuf[STARCH_BUFFER_MAX_LENGTH] = {0};
+ unsigned char *retransformBuf = NULL;
int64_t nRetransformBuf = 0;
+
+ /* allocate memory for intermediate buffer variables */
+ retransformLineBuf = (unsigned char *) calloc(TOKENS_MAX_LENGTH, sizeof(unsigned char));
+ if (!retransformLineBuf) {
+ fprintf(stderr, "ERROR: Insufficient memory for intermediate buffer variable [retransformLineBuf].\n");
+ exit(ENOMEM);
+ }
+ retransformBuf = (unsigned char *) calloc(STARCH_BUFFER_MAX_LENGTH, sizeof(unsigned char));
+ if (!retransformBuf) {
+ fprintf(stderr, "ERROR: Insufficient memory for intermediate buffer variable [retransformBuf].\n");
+ exit(ENOMEM);
+ }
/* bzip2 variables */
BZFILE *bzInFp = NULL;
BZFILE *bzOutFp = NULL;
int bzInError = BZ_OK;
int bzOutError = BZ_OK;
- unsigned char bzReadBuf[STARCH_BZ_BUFFER_MAX_LENGTH] = {0};
+ unsigned char *bzReadBuf = NULL;
size_t nBzReadBuf = STARCH_BZ_BUFFER_MAX_LENGTH;
- unsigned char bzRemainderBuf[STARCH_BZ_BUFFER_MAX_LENGTH] = {0};
+ unsigned char *bzRemainderBuf = NULL;
size_t nBzRemainderBuf = 0;
size_t nBzRead = 0;
size_t bzBufIndex = 0;
- unsigned char bzLineBuf[STARCH_BZ_BUFFER_MAX_LENGTH] = {0};
+ unsigned char *bzLineBuf = NULL;
unsigned int bzOutBytesConsumed = 0U;
unsigned int bzOutBytesWritten = 0U;
+ /* allocate memory for bzip2 variables */
+ bzReadBuf = (unsigned char *) calloc(STARCH_BZ_BUFFER_MAX_LENGTH, sizeof(unsigned char));
+ if (!bzReadBuf) {
+ fprintf(stderr, "ERROR: Insufficient memory for bzip2 variable [bzReadBuf].\n");
+ exit(ENOMEM);
+ }
+ bzRemainderBuf = (unsigned char *) calloc(STARCH_BZ_BUFFER_MAX_LENGTH, sizeof(unsigned char));
+ if (!bzRemainderBuf) {
+ fprintf(stderr, "ERROR: Insufficient memory for bzip2 variable [bzRemainderBuf].\n");
+ exit(ENOMEM);
+ }
+ bzLineBuf = (unsigned char *) calloc(STARCH_BZ_BUFFER_MAX_LENGTH, sizeof(unsigned char));
+ if (!bzLineBuf) {
+ fprintf(stderr, "ERROR: Insufficient memory for bzip2 variable [bzLineBuf].\n");
+ exit(ENOMEM);
+ }
+
/* gzip variables */
z_stream zInStream;
z_stream zOutStream;
int zInError = -1;
int zOutError = -1;
- unsigned char zOutBuffer[STARCH_Z_BUFFER_MAX_LENGTH] = {0};
- unsigned char zReadBuf[STARCH_Z_CHUNK/1024];
- unsigned char zOutBuf[STARCH_Z_CHUNK];
+ unsigned char *zOutBuffer = NULL;
+ unsigned char *zReadBuf = NULL;
+ unsigned char *zOutBuf = NULL;
size_t zInHave = 0;
size_t zOutHave = 0;
size_t zBufIndex = 0;
- unsigned char zRemainderBuf[STARCH_Z_BUFFER_MAX_LENGTH] = {0};
+ unsigned char *zRemainderBuf = NULL;
size_t nZRemainderBuf = 0;
- unsigned char zLineBuf[STARCH_Z_BUFFER_MAX_LENGTH] = {0};
+ unsigned char *zLineBuf = NULL;
+
+ /* allocate memory for gzip variables */
+ zOutBuffer = (unsigned char *) calloc(STARCH_Z_BUFFER_MAX_LENGTH, sizeof(unsigned char));
+ if (!zOutBuffer) {
+ fprintf(stderr, "ERROR: Insufficient memory for gzip variable [zOutBuffer].\n");
+ exit(ENOMEM);
+ }
+ zReadBuf = (unsigned char *) calloc(STARCH_Z_CHUNK/1024, sizeof(unsigned char));
+ if (!zReadBuf) {
+ fprintf(stderr, "ERROR: Insufficient memory for gzip variable [zReadBuf].\n");
+ exit(ENOMEM);
+ }
+ zOutBuf = (unsigned char *) calloc(STARCH_Z_CHUNK, sizeof(unsigned char));
+ if (!zOutBuf) {
+ fprintf(stderr, "ERROR: Insufficient memory for gzip variable [zOutBuf].\n");
+ exit(ENOMEM);
+ }
+ zRemainderBuf = (unsigned char *) calloc(STARCH_Z_BUFFER_MAX_LENGTH, sizeof(unsigned char));
+ if (!zRemainderBuf) {
+ fprintf(stderr, "ERROR: Insufficient memory for gzip variable [zRemainderBuf].\n");
+ exit(ENOMEM);
+ }
+ zLineBuf = (unsigned char *) calloc(STARCH_Z_BUFFER_MAX_LENGTH, sizeof(unsigned char));
+ if (!zLineBuf) {
+ fprintf(stderr, "ERROR: Insufficient memory for gzip variable [zLineBuf].\n");
+ exit(ENOMEM);
+ }
/* transformation variables */
size_t lastNewlineOffset = 0U;
@@ -769,8 +869,8 @@ STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, con
SignedCoordType t_start = 0;
SignedCoordType t_pLength = 0;
SignedCoordType t_lastEnd = 0;
- char t_firstInputToken[UNSTARCH_FIRST_TOKEN_MAX_LENGTH] = {0};
- char t_secondInputToken[UNSTARCH_SECOND_TOKEN_MAX_LENGTH] = {0};
+ char *t_firstInputToken = NULL;
+ char *t_secondInputToken = NULL;
char *t_currChr = NULL;
size_t t_currChrLen = 0U;
SignedCoordType t_currStart = 0;
@@ -788,6 +888,24 @@ STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, con
Boolean t_duplicateElementExists = STARCH_DEFAULT_DUPLICATE_ELEMENT_FLAG_VALUE;
Boolean t_nestedElementExists = STARCH_DEFAULT_NESTED_ELEMENT_FLAG_VALUE;
size_t t_fileSize = 0U;
+ LineLengthType t_lineMaxStringLength = STARCH_DEFAULT_LINE_STRING_LENGTH;
+
+ /* allocate memory for transformation variables */
+ t_firstInputToken = (char *) calloc(UNSTARCH_FIRST_TOKEN_MAX_LENGTH, sizeof(char));
+ if (!t_firstInputToken) {
+ fprintf(stderr, "ERROR: Insufficient memory for transformation variable [t_firstInputToken].\n");
+ exit(ENOMEM);
+ }
+ t_secondInputToken = (char *) calloc(UNSTARCH_SECOND_TOKEN_MAX_LENGTH, sizeof(char));
+ if (!t_secondInputToken) {
+ fprintf(stderr, "ERROR: Insufficient memory for transformation variable [t_secondInputToken].\n");
+ exit(ENOMEM);
+ }
+
+ /* hash variables */
+ struct sha1_ctx t_perChromosomeHashCtx;
+ unsigned char t_sha1Digest[STARCH2_MD_FOOTER_SHA1_LENGTH] = {0};
+ char *t_base64EncodedSha1Digest = NULL;
static const char tab = '\t';
@@ -850,6 +968,11 @@ STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, con
fseeko(inFp, (off_t) startOffset, SEEK_SET);
#endif
+ if (generatePerChrSignatureFlag) {
+ /* set up per-chromosome hash context */
+ sha1_init_ctx(&t_perChromosomeHashCtx);
+ }
+
/*
Set up I/O streams
*/
@@ -1031,6 +1154,12 @@ STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, con
*/
if (bufChar == '\n') {
bzLineBuf[bufCharIndex - 1] = '\0';
+#ifdef __cplusplus
+ t_lineMaxStringLength = (t_lineMaxStringLength >= static_cast<LineLengthType>(bufCharIndex - 1)) ? t_lineMaxStringLength : static_cast<LineLengthType>(bufCharIndex - 1);
+#else
+ t_lineMaxStringLength = (t_lineMaxStringLength >= (LineLengthType)(bufCharIndex - 1)) ? t_lineMaxStringLength : (LineLengthType)(bufCharIndex - 1);
+#endif
+
#ifdef DEBUG
/*
fprintf(stderr, "\tbzLineBuf: %s\n", bzLineBuf);
@@ -1038,19 +1167,19 @@ STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, con
*/
#endif
UNSTARCH_extractRawLine( inChr,
- bzLineBuf,
- tab,
- &t_start,
- &t_pLength,
- &t_lastEnd,
- t_firstInputToken,
- t_secondInputToken,
- &t_currChr,
- &t_currChrLen,
- &t_currStart,
- &t_currStop,
- &t_currRemainder,
- &t_currRemainderLen );
+ bzLineBuf,
+ tab,
+ &t_start,
+ &t_pLength,
+ &t_lastEnd,
+ t_firstInputToken,
+ t_secondInputToken,
+ &t_currChr,
+ &t_currChrLen,
+ &t_currStart,
+ &t_currStop,
+ &t_currRemainder,
+ &t_currRemainderLen );
#ifdef DEBUG
/*
@@ -1087,14 +1216,18 @@ STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, con
#else
UNSTARCH_reverseTransformCoordinates( (const LineCountType) t_lineIdx,
#endif
- &t_lastPosition,
- &t_lcDiff,
- &t_currStart,
- &t_currStop,
- &t_currRemainder,
- retransformLineBuf,
- &nRetransformLineBuf,
- &nRetransformLineBufPos );
+ &t_lastPosition,
+ &t_lcDiff,
+ &t_currStart,
+ &t_currStop,
+ &t_currRemainder,
+ retransformLineBuf,
+ &nRetransformLineBuf,
+ &nRetransformLineBufPos );
+
+ if ((reportProgressFlag == kStarchTrue) && (t_lineIdx % reportProgressN == 0)) {
+ fprintf(stderr, "PROGRESS: Retransforming element [%" PRIu64 "] of chromosome [%s] -> [%s\t%" PRId64 "\t%" PRId64 "\t%s]\n", t_lineIdx, t_currChr, t_currChr, t_currStart, t_currStop, t_currRemainder);
+ }
/* adjust per-stream statistics */
t_lastPosition = t_currStop;
@@ -1143,6 +1276,16 @@ STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, con
outbound compression type, add retransformLineBuf to the start of
retransformBuf, and set nRetransformBuf.
*/
+
+ if (generatePerChrSignatureFlag) {
+ /* hash the transformed buffer */
+#ifdef __cplusplus
+ sha1_process_bytes(retransformBuf, static_cast<size_t>( nRetransformBuf ), &t_perChromosomeHashCtx);
+#else
+ sha1_process_bytes(retransformBuf, (size_t) nRetransformBuf, &t_perChromosomeHashCtx);
+#endif
+ }
+
switch (outType) {
/* compress with bzip2 */
case kBzip2: {
@@ -1288,6 +1431,15 @@ STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, con
}
}
+ if (generatePerChrSignatureFlag) {
+ /* hash the transformed buffer */
+#ifdef __cplusplus
+ sha1_process_bytes(retransformBuf, static_cast<size_t>( nRetransformBuf ), &t_perChromosomeHashCtx);
+#else
+ sha1_process_bytes(retransformBuf, (size_t) nRetransformBuf, &t_perChromosomeHashCtx);
+#endif
+ }
+
/* compress whatever is left in the retransform buffer */
switch (outType) {
case kBzip2: {
@@ -1506,40 +1658,49 @@ STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, con
if (bufChar == '\n') {
zLineBuf[bufCharIndex - 1] = '\0';
-
+#ifdef __cplusplus
+ t_lineMaxStringLength = (t_lineMaxStringLength >= static_cast<LineLengthType>(bufCharIndex - 1)) ? t_lineMaxStringLength : static_cast<LineLengthType>(bufCharIndex - 1);
+#else
+ t_lineMaxStringLength = (t_lineMaxStringLength >= (LineLengthType)(bufCharIndex - 1)) ? t_lineMaxStringLength : (LineLengthType)(bufCharIndex - 1);
+#endif
/* extract a line of transformed data */
- UNSTARCH_extractRawLine(inChr,
- zLineBuf,
- tab,
- &t_start,
- &t_pLength,
- &t_lastEnd,
- t_firstInputToken,
- t_secondInputToken,
- &t_currChr,
- &t_currChrLen,
- &t_currStart,
- &t_currStop,
- &t_currRemainder,
- &t_currRemainderLen);
+ UNSTARCH_extractRawLine( inChr,
+ zLineBuf,
+ tab,
+ &t_start,
+ &t_pLength,
+ &t_lastEnd,
+ t_firstInputToken,
+ t_secondInputToken,
+ &t_currChr,
+ &t_currChrLen,
+ &t_currStart,
+ &t_currStop,
+ &t_currRemainder,
+ &t_currRemainderLen );
if (zLineBuf[0] != 'p') {
/* increment line counter */
t_lineIdx++;
/* reverse transform */
#ifdef __cplusplus
- UNSTARCH_reverseTransformCoordinates(static_cast<const LineCountType>( t_lineIdx ),
+ UNSTARCH_reverseTransformCoordinates( static_cast<const LineCountType>( t_lineIdx ),
#else
- UNSTARCH_reverseTransformCoordinates((const LineCountType) t_lineIdx,
+ UNSTARCH_reverseTransformCoordinates( (const LineCountType) t_lineIdx,
#endif
- &t_lastPosition,
- &t_lcDiff,
- &t_currStart,
- &t_currStop,
- &t_currRemainder,
- retransformLineBuf,
- &nRetransformLineBuf,
- &nRetransformLineBufPos);
+ &t_lastPosition,
+ &t_lcDiff,
+ &t_currStart,
+ &t_currStop,
+ &t_currRemainder,
+ retransformLineBuf,
+ &nRetransformLineBuf,
+ &nRetransformLineBufPos );
+
+ if ((reportProgressFlag == kStarchTrue) && (t_lineIdx % reportProgressN == 0)) {
+ fprintf(stderr, "PROGRESS: Retransforming element [%" PRIu64 "] of chromosome [%s] -> [%s\t%" PRId64 "\t%" PRId64 "\t%s]\n", t_lineIdx, t_currChr, t_currChr, t_currStart, t_currStop, t_currRemainder);
+ }
+
/* adjust statistics */
t_lastPosition = t_currStop;
#ifdef __cplusplus
@@ -1583,6 +1744,16 @@ STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, con
nRetransformLineBuf = 0;
}
else {
+
+ if (generatePerChrSignatureFlag) {
+ /* hash the transformed buffer */
+#ifdef __cplusplus
+ sha1_process_bytes(retransformBuf, static_cast<size_t>( nRetransformBuf ), &t_perChromosomeHashCtx);
+#else
+ sha1_process_bytes(retransformBuf, (size_t) nRetransformBuf, &t_perChromosomeHashCtx);
+#endif
+ }
+
/*
Compress whatever is already in retransformBuf given the specified
outbound compression type, add retransformLineBuf to the start of
@@ -1702,7 +1873,16 @@ STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, con
}
} while (zInStream.avail_out == 0);
} while (zInError != Z_STREAM_END);
-
+
+ if (generatePerChrSignatureFlag) {
+ /* hash the transformed buffer */
+#ifdef __cplusplus
+ sha1_process_bytes(retransformBuf, static_cast<size_t>( nRetransformBuf ), &t_perChromosomeHashCtx);
+#else
+ sha1_process_bytes(retransformBuf, (size_t) nRetransformBuf, &t_perChromosomeHashCtx);
+#endif
+ }
+
/* process any remainder */
switch (outType) {
case kBzip2: {
@@ -1786,6 +1966,24 @@ STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, con
}
}
+ if (generatePerChrSignatureFlag) {
+ sha1_finish_ctx(&t_perChromosomeHashCtx, t_sha1Digest);
+#ifdef __cplusplus
+ STARCH_encodeBase64(&t_base64EncodedSha1Digest,
+ static_cast<size_t>( STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH ),
+ reinterpret_cast<const unsigned char *>( t_sha1Digest ),
+ static_cast<size_t>( STARCH2_MD_FOOTER_SHA1_LENGTH ) );
+#else
+ STARCH_encodeBase64(&t_base64EncodedSha1Digest,
+ (const size_t) STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH,
+ (const unsigned char *) t_sha1Digest,
+ (const size_t) STARCH2_MD_FOOTER_SHA1_LENGTH);
+#endif
+#ifdef DEBUG
+ fprintf(stderr, "\nPROGRESS: SHA-1 digest for chr [%s] is [%s]\n", inChr, t_base64EncodedSha1Digest);
+#endif
+ }
+
/* clean up outbound compression stream */
switch (outType) {
case kBzip2: {
@@ -1841,24 +2039,6 @@ STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, con
}
}
-#ifdef DEBUG
- if ((t_currStart > 5238450) && (t_currStart < 5238520)) {
- fprintf(stderr, "\n---\nmetadata\n---\n");
- fprintf(stderr, "\tinChr -> %s\n", inChr);
- fprintf(stderr, "\toutTagFn -> %s\n", outTagFn);
- fprintf(stderr, "\tt_fileSize -> %zu\n", t_fileSize);
-#ifdef __cplusplus
- fprintf(stderr, "\tt_lineIdx -> %" PRIu64 "\n", static_cast<uint64_t>( t_lineIdx ));
- fprintf(stderr, "\tt_totalNonUniqueBases -> %" PRIu64 "\n", static_cast<uint64_t>( t_totalNonUniqueBases ));
- fprintf(stderr, "\tt_totalUniqueBases -> %" PRIu64 "\n", static_cast<uint64_t>( t_totalUniqueBases ));
-#else
- fprintf(stderr, "\tt_lineIdx -> %" PRIu64 "\n", (uint64_t) t_lineIdx);
- fprintf(stderr, "\tt_totalNonUniqueBases -> %" PRIu64 "\n", (uint64_t) t_totalNonUniqueBases);
- fprintf(stderr, "\tt_totalUniqueBases -> %" PRIu64 "\n", (uint64_t) t_totalUniqueBases);
-#endif
- }
-#endif
-
/* update the metadata */
if (!*outMd)
#ifdef __cplusplus
@@ -1872,7 +2052,9 @@ STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, con
t_totalNonUniqueBases,
t_totalUniqueBases,
t_duplicateElementExists,
- t_nestedElementExists );
+ t_nestedElementExists,
+ t_base64EncodedSha1Digest,
+ t_lineMaxStringLength );
else
*outMd = STARCH_addMetadata( *outMd,
#ifdef __cplusplus
@@ -1886,11 +2068,25 @@ STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, con
t_totalNonUniqueBases,
t_totalUniqueBases,
t_duplicateElementExists,
- t_nestedElementExists );
+ t_nestedElementExists,
+ t_base64EncodedSha1Digest,
+ t_lineMaxStringLength );
/* cleanup */
- if (outTagFn)
- free(outTagFn), outTagFn = NULL;
+ free(outTagFn), outTagFn = NULL;
+ free(t_base64EncodedSha1Digest), t_base64EncodedSha1Digest = NULL;
+ free(retransformLineBuf), retransformLineBuf = NULL;
+ free(retransformBuf), retransformBuf = NULL;
+ free(bzReadBuf), bzReadBuf = NULL;
+ free(bzRemainderBuf), bzRemainderBuf = NULL;
+ free(bzLineBuf), bzLineBuf = NULL;
+ free(zOutBuffer), zOutBuffer = NULL;
+ free(zReadBuf), zReadBuf = NULL;
+ free(zOutBuf), zOutBuf = NULL;
+ free(zRemainderBuf), zRemainderBuf = NULL;
+ free(zLineBuf), zLineBuf = NULL;
+ free(t_firstInputToken), t_firstInputToken = NULL;
+ free(t_secondInputToken), t_secondInputToken = NULL;
return STARCHCAT_EXIT_SUCCESS;
}
@@ -1940,18 +2136,18 @@ STARCHCAT_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, cons
case kBzip2: {
#ifdef __cplusplus
if ( UNSTARCH_extractDataWithBzip2( &inFp,
- uncomprOutFnPtr,
- reinterpret_cast<const char *>( inChr ),
- reinterpret_cast<const Metadata *>( inMd ),
- static_cast<const uint64_t>( inRec->mdOffset ),
- static_cast<const Boolean>( inRec->hFlag )) != 0 ) {
+ uncomprOutFnPtr,
+ reinterpret_cast<const char *>( inChr ),
+ reinterpret_cast<const Metadata *>( inMd ),
+ static_cast<const uint64_t>( inRec->mdOffset ),
+ static_cast<const Boolean>( inRec->hFlag )) != 0 ) {
#else
if ( UNSTARCH_extractDataWithBzip2( &inFp,
- uncomprOutFnPtr,
- (const char *) inChr,
- (const Metadata *) inMd,
- (const uint64_t) inRec->mdOffset,
- (const Boolean) inRec->hFlag) != 0 ) {
+ uncomprOutFnPtr,
+ (const char *) inChr,
+ (const Metadata *) inMd,
+ (const uint64_t) inRec->mdOffset,
+ (const Boolean) inRec->hFlag) != 0 ) {
#endif
fprintf(stderr, "ERROR: Could not extract to uncompressed output file (bzip2).\n");
return STARCHCAT_EXIT_FAILURE;
@@ -1961,18 +2157,18 @@ STARCHCAT_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, cons
case kGzip: {
#ifdef __cplusplus
if ( UNSTARCH_extractDataWithGzip( &inFp,
- uncomprOutFnPtr,
- reinterpret_cast<const char *>( inChr ),
- reinterpret_cast<const Metadata *>( inMd ),
- static_cast<const uint64_t>( inRec->mdOffset ),
- static_cast<const Boolean>( inRec->hFlag )) != 0 ) {
+ uncomprOutFnPtr,
+ reinterpret_cast<const char *>( inChr ),
+ reinterpret_cast<const Metadata *>( inMd ),
+ static_cast<const uint64_t>( inRec->mdOffset ),
+ static_cast<const Boolean>( inRec->hFlag )) != 0 ) {
#else
if ( UNSTARCH_extractDataWithGzip( &inFp,
- uncomprOutFnPtr,
- (const char *) inChr,
- (const Metadata *) inMd,
- (const uint64_t) inRec->mdOffset,
- (const Boolean) inRec->hFlag) != 0 ) {
+ uncomprOutFnPtr,
+ (const char *) inChr,
+ (const Metadata *) inMd,
+ (const uint64_t) inRec->mdOffset,
+ (const Boolean) inRec->hFlag) != 0 ) {
#endif
fprintf(stderr, "ERROR: Could not extract to uncompressed output file (gzip).\n");
return STARCHCAT_EXIT_FAILURE;
@@ -1997,21 +2193,21 @@ STARCHCAT_rewriteInputRecordToOutput (Metadata **outMd, const char *outTag, cons
}
#ifdef __cplusplus
if ( STARCH_transformHeaderlessInput( &(*outMd),
- reinterpret_cast<const FILE *>( transformFnPtr ),
- static_cast<const CompressionType>( outType ),
- reinterpret_cast<const char *>( outTag ),
- static_cast<const Boolean>( kStarchFinalizeTransformFalse ),
- reinterpret_cast<const char *>( note ) ) != 0 ) {
+ reinterpret_cast<const FILE *>( transformFnPtr ),
+ static_cast<const CompressionType>( outType ),
+ reinterpret_cast<const char *>( outTag ),
+ static_cast<const Boolean>( kStarchFinalizeTransformFalse ),
+ reinterpret_cast<const char *>( note ) ) != 0 ) {
fprintf(stderr, "ERROR: Could not transform output file\n");
return STARCHCAT_EXIT_FAILURE;
}
#else
if ( STARCH_transformHeaderlessInput( &(*outMd),
- (const FILE *) transformFnPtr,
- (const CompressionType) outType,
- (const char *) outTag,
- (const Boolean) kStarchFinalizeTransformFalse,
- (const char *) note ) != 0 ) {
+ (const FILE *) transformFnPtr,
+ (const CompressionType) outType,
+ (const char *) outTag,
+ (const Boolean) kStarchFinalizeTransformFalse,
+ (const char *) note ) != 0 ) {
fprintf(stderr, "ERROR: Could not transform output file\n");
return STARCHCAT_EXIT_FAILURE;
}
@@ -2179,6 +2375,124 @@ STARCHCAT2_identifyLowestBedElement (const Boolean *eobFlags, const SignedCoordT
return STARCHCAT_EXIT_SUCCESS;
}
+int
+STARCHCAT2_identifyLowestBedElementV2p2 (const Boolean *eobFlags, const SignedCoordType *starts, const SignedCoordType *stops, const char **remainders, const size_t numRecords, size_t *lowestIdx)
+{
+#ifdef DEBUG
+ fprintf (stderr, "\n--- STARCHCAT2_identifyLowestBedElementV2p2() ---\n");
+#endif
+ size_t recIdx;
+
+#ifdef __cplusplus
+ SignedCoordType currentStart = std::numeric_limits<int64_t>::min();
+ SignedCoordType currentStop = std::numeric_limits<int64_t>::min();
+ SignedCoordType lowestStart = std::numeric_limits<int64_t>::max();
+ SignedCoordType lowestStop = std::numeric_limits<int64_t>::max();
+#else
+ SignedCoordType currentStart = INT64_MIN;
+ SignedCoordType currentStop = INT64_MIN;
+ SignedCoordType lowestStart = INT64_MAX;
+ SignedCoordType lowestStop = INT64_MAX;
+#endif
+ Boolean checkStopFlag = kStarchFalse;
+
+#ifdef __cplusplus
+ *lowestIdx = static_cast<size_t>( -1 );
+#else
+ *lowestIdx = (size_t) -1;
+#endif
+
+ for (recIdx = 0U; recIdx < numRecords; recIdx++) {
+ if (eobFlags[recIdx] == kStarchFalse) {
+ currentStart = starts[recIdx];
+ currentStop = stops[recIdx];
+#ifdef DEBUG
+ fprintf(stderr, "\trecIdx -> %zu \n", recIdx);
+ fprintf(stderr, "\tstarts[recIdx] -> %" PRId64 " \n", currentStart);
+ fprintf(stderr, "\tstops[recIdx] -> %" PRId64 " \n", currentStop);
+ fprintf(stderr, "\tlowestStart -> %" PRId64 "\n", lowestStart);
+#endif
+ if (currentStart < lowestStart) {
+ lowestStart = currentStart;
+ *lowestIdx = recIdx;
+ }
+ else if (currentStart == lowestStart) {
+ checkStopFlag = kStarchTrue;
+ }
+ }
+ }
+
+ /*
+ if the start coordinates match, we decide upon comparison of the stop coordinates
+ by keeping an array of matches on lowest-stop coordinates -- when there are no more
+ records to check, then compare remainders between matching lowest-stop records
+ */
+
+ if (checkStopFlag == kStarchTrue) {
+
+ size_t *lowestStopMatches = NULL;
+ size_t lowestStopMatchesCapacity = 0;
+#ifdef __cplusplus
+ lowestStopMatches = static_cast<size_t *> (malloc(numRecords * sizeof(*lowestStopMatches)));
+#else
+ lowestStopMatches = malloc(numRecords * sizeof(*lowestStopMatches));
+#endif
+ if (!lowestStopMatches) {
+ fprintf(stderr, "ERROR: Could not allocate space for lowest-stop matches array!\n");
+ exit(ENOMEM);
+ }
+
+ for (recIdx = 0U; recIdx < numRecords; recIdx++) {
+ currentStart = starts[recIdx];
+ currentStop = stops[recIdx];
+ if (eobFlags[recIdx] == kStarchFalse) {
+ if ((currentStart == lowestStart) && (currentStop < lowestStop)) {
+ lowestStop = currentStop;
+ *lowestIdx = recIdx;
+ /* empty out the lowestStopMatches array and add this element */
+ lowestStopMatchesCapacity = 0;
+ lowestStopMatches[lowestStopMatchesCapacity++] = recIdx;
+ }
+ else if ((currentStart == lowestStart) && (currentStop == lowestStop)) {
+ lowestStopMatches[lowestStopMatchesCapacity++] = recIdx;
+ }
+ }
+ }
+
+ /*
+ process remainders when there are two or more lowestStopMatches entries
+ (the lowestStopMatches array points to record indices, so we dereference
+ from there to the original index)
+ */
+
+ size_t lowestStopMatchIdx;
+ if (lowestStopMatchesCapacity > 1) {
+ const char *lowestRemainder = remainders[lowestStopMatches[0]];
+ for (lowestStopMatchIdx = 1U; lowestStopMatchIdx < lowestStopMatchesCapacity; lowestStopMatchIdx++) {
+ if (strcmp(lowestRemainder, remainders[lowestStopMatches[lowestStopMatchIdx]]) > 0) {
+ *lowestIdx = lowestStopMatches[lowestStopMatchIdx];
+ }
+ }
+ }
+
+ free(lowestStopMatches), lowestStopMatches = NULL;
+ }
+
+#ifdef DEBUG
+ fprintf(stderr, "\tLE STATE *lowestIdx -> %zu\n", *lowestIdx);
+#endif
+
+#ifdef __cplusplus
+ if (*lowestIdx == static_cast<size_t>( -1 ))
+ return STARCHCAT_EXIT_FAILURE;
+#else
+ if (*lowestIdx == (size_t) -1)
+ return STARCHCAT_EXIT_FAILURE;
+#endif
+
+ return STARCHCAT_EXIT_SUCCESS;
+}
+
int
STARCHCAT2_pullNextBedElement (const size_t recIdx, const char **inLinesBuf, const LineCountType *nInLinesBuf, char **outLineBuf, uint64_t **inBufNewlineOffsets)
{
@@ -2262,6 +2576,8 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
char **extractedElements = NULL;
SignedCoordType *starts = NULL;
SignedCoordType *stops = NULL;
+ char **remainders = NULL;
+ size_t remainderCapacity = TOKENS_MAX_LENGTH + 1;
size_t lowestStartElementIdx = 0U;
Boolean *eobFlags = NULL;
Boolean *eofFlags = NULL;
@@ -2278,10 +2594,12 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
uint64_t bzOutBytesWritten = 0;
uint64_t finalStreamSize = 0;
LineCountType finalLineCount = 0;
+ LineLengthType finalLineMaxStringLength = STARCH_DEFAULT_LINE_STRING_LENGTH;
BaseCountType finalTotalNonUniqueBases = 0;
BaseCountType finalTotalUniqueBases = 0;
Boolean finalDuplicateElementExists = STARCH_DEFAULT_DUPLICATE_ELEMENT_FLAG_VALUE;
Boolean finalNestedElementExists = STARCH_DEFAULT_NESTED_ELEMENT_FLAG_VALUE;
+ char *finalSignature = NULL;
char *finalOutTagFn = NULL;
MetadataRecord *inRecord = NULL;
size_t *nBzReads = NULL;
@@ -2293,6 +2611,11 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
size_t nCompressionBuffer = STARCHCAT_RETRANSFORM_LINE_COUNT_MAX * TOKENS_MAX_LENGTH + 1;
int lowestElementRes = STARCHCAT_EXIT_SUCCESS;
+ /* hash variables */
+ struct sha1_ctx r_perChromosomeHashCtx;
+ unsigned char r_sha1Digest[STARCH2_MD_FOOTER_SHA1_LENGTH];
+ char *r_base64EncodedSha1Digest = NULL;
+
/* setup */
#ifdef __cplusplus
extractionBuffers = static_cast<char **>( malloc(sizeof(char *) * summary->numRecords) );
@@ -2305,6 +2628,7 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
eofFlags = static_cast<Boolean *>( malloc(sizeof(Boolean) * summary->numRecords) );
starts = static_cast<SignedCoordType *>( malloc(sizeof(SignedCoordType) * summary->numRecords) );
stops = static_cast<SignedCoordType *>( malloc(sizeof(SignedCoordType) * summary->numRecords) );
+ remainders = static_cast<char **>( malloc(sizeof(char *) * summary->numRecords) );
transformStates = static_cast<TransformState **>( malloc(sizeof(TransformState *) * summary->numRecords) );
extractionRemainderBufs = static_cast<char **>( malloc(sizeof(char *) * summary->numRecords) );
nExtractionRemainderBufs = static_cast<size_t *>( malloc(sizeof(size_t) * summary->numRecords) );
@@ -2326,6 +2650,7 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
eofFlags = malloc(sizeof(Boolean) * summary->numRecords);
starts = malloc(sizeof(SignedCoordType) * summary->numRecords);
stops = malloc(sizeof(SignedCoordType) * summary->numRecords);
+ remainders = malloc(sizeof(char *) * summary->numRecords);
transformStates = malloc(sizeof(TransformState *) * summary->numRecords);
extractionRemainderBufs = malloc(sizeof(char *) * summary->numRecords);
nExtractionRemainderBufs = malloc(sizeof(size_t) * summary->numRecords);
@@ -2340,7 +2665,10 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
memset(outputRetransformState->r_chromosome, 0, TOKEN_CHR_MAX_LENGTH);
- /* test if we allocated memory - TO-DO */
+ /* set up per-chromosome hash context */
+ sha1_init_ctx (&r_perChromosomeHashCtx);
+
+ /* test if we allocated memory */
if (!compressionBuffer) {
fprintf(stderr, "ERROR: Could not allocate space for compression buffer!\n");
return STARCHCAT_EXIT_FAILURE;
@@ -2442,29 +2770,29 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
nBzReads[inRecIdx] = 0;
#ifdef __cplusplus
if (STARCHCAT2_setupBzip2InputStream(static_cast<const size_t>( inRecIdx ),
- summary,
- &bzInFps[inRecIdx]) != STARCHCAT_EXIT_SUCCESS) {
+ summary,
+ &bzInFps[inRecIdx]) != STARCHCAT_EXIT_SUCCESS) {
#else
if (STARCHCAT2_setupBzip2InputStream((const size_t) inRecIdx,
- summary,
- &bzInFps[inRecIdx]) != STARCHCAT_EXIT_SUCCESS) {
+ summary,
+ &bzInFps[inRecIdx]) != STARCHCAT_EXIT_SUCCESS) {
#endif
fprintf(stderr, "ERROR: Could not set up bzip2 input stream at index [%zu]!\n", inRecIdx);
return STARCHCAT_EXIT_FAILURE;
}
if (STARCHCAT2_fillExtractionBufferFromBzip2Stream(&eofFlags[inRecIdx],
#ifdef __cplusplus
- const_cast<char *>( inChr ),
+ const_cast<char *>( inChr ),
#else
- (char *) inChr,
+ (char *) inChr,
#endif
- extractionBuffers[inRecIdx],
- &nExtractionBuffers[inRecIdx],
- &bzInFps[inRecIdx],
- &nBzReads[inRecIdx],
- extractionRemainderBufs[inRecIdx],
- &nExtractionRemainderBufs[inRecIdx],
- transformStates[inRecIdx]) != STARCHCAT_EXIT_SUCCESS) {
+ extractionBuffers[inRecIdx],
+ &nExtractionBuffers[inRecIdx],
+ &bzInFps[inRecIdx],
+ &nBzReads[inRecIdx],
+ extractionRemainderBufs[inRecIdx],
+ &nExtractionRemainderBufs[inRecIdx],
+ transformStates[inRecIdx]) != STARCHCAT_EXIT_SUCCESS) {
fprintf(stderr, "ERROR: Could not extract data from bzip2 input stream at index [%zu]!\n", inRecIdx);
return STARCHCAT_EXIT_FAILURE;
}
@@ -2477,19 +2805,19 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
return STARCHCAT_EXIT_FAILURE;
}
if (STARCHCAT2_fillExtractionBufferFromGzipStream(&eofFlags[inRecIdx],
- &zInFps[inRecIdx],
+ &zInFps[inRecIdx],
#ifdef __cplusplus
- const_cast<char *>( inChr ),
+ const_cast<char *>( inChr ),
#else
- (char *) inChr,
+ (char *) inChr,
#endif
- extractionBuffers[inRecIdx],
- &nExtractionBuffers[inRecIdx],
- &zInStreams[inRecIdx],
- &nZReads[inRecIdx],
- &extractionRemainderBufs[inRecIdx],
- &nExtractionRemainderBufs[inRecIdx],
- transformStates[inRecIdx]) != STARCHCAT_EXIT_SUCCESS) {
+ extractionBuffers[inRecIdx],
+ &nExtractionBuffers[inRecIdx],
+ &zInStreams[inRecIdx],
+ &nZReads[inRecIdx],
+ &extractionRemainderBufs[inRecIdx],
+ &nExtractionRemainderBufs[inRecIdx],
+ transformStates[inRecIdx]) != STARCHCAT_EXIT_SUCCESS) {
fprintf(stderr, "ERROR: Could not extract data from gzip input stream at index [%zu]!\n", inRecIdx);
return STARCHCAT_EXIT_FAILURE;
}
@@ -2514,9 +2842,9 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
/* eobFlags[inRecIdx] = kStarchFalse; */
STARCHCAT2_extractBedLine(&eobFlags[inRecIdx],
- extractionBuffers[inRecIdx],
- &extractionBufferOffsets[inRecIdx],
- &extractedElements[inRecIdx]);
+ extractionBuffers[inRecIdx],
+ &extractionBufferOffsets[inRecIdx],
+ &extractedElements[inRecIdx]);
extractedLineCounts[inRecIdx]--;
/* memset(outputRetransformState->r_chromosome, 0, TOKEN_CHR_MAX_LENGTH); */
@@ -2531,6 +2859,8 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
outputRetransformState->r_lastPosition = 0;
outputRetransformState->r_totalNonUniqueBases = 0;
outputRetransformState->r_totalUniqueBases = 0;
+ outputRetransformState->r_signature = NULL;
+ outputRetransformState->r_lineMaxStringLength = STARCH_DEFAULT_LINE_STRING_LENGTH;
outputRetransformState->r_duplicateElementExists = STARCH_DEFAULT_DUPLICATE_ELEMENT_FLAG_VALUE;
outputRetransformState->r_nestedElementExists = STARCH_DEFAULT_NESTED_ELEMENT_FLAG_VALUE;
outputRetransformState->r_previousStop = 0;
@@ -2544,6 +2874,17 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
starts[inRecIdx] = 0;
stops[inRecIdx] = 0;
+
+#ifdef __cplusplus
+ remainders[inRecIdx] = static_cast<char *>( malloc(remainderCapacity) );
+#else
+ remainders[inRecIdx] = malloc(remainderCapacity);
+#endif
+ if (!remainders[inRecIdx]) {
+ fprintf(stderr, "ERROR: Could not allocate memory for remainder buffer!\n");
+ return STARCHCAT_EXIT_FAILURE;
+ }
+ memset(remainders[inRecIdx], 0, remainderCapacity);
}
/* merge */
@@ -2558,55 +2899,67 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
}
/* 2, 3 -- parse coordinates for each record's current bed element (this gets us the start and stop coords) */
+
for (inRecIdx = 0U; inRecIdx < summary->numRecords; inRecIdx++) {
#ifdef DEBUG
fprintf(stderr, "inRecIdx -> %zu\teobFlags[inRecIdx] -> %d\teofFlags[inRecIdx] -> %d\n", inRecIdx, eobFlags[inRecIdx], eofFlags[inRecIdx]);
#endif
//if ((eobFlags[inRecIdx] == kStarchFalse) && (eofFlags[inRecIdx] == kStarchFalse))
- if (eobFlags[inRecIdx] == kStarchFalse)
+ if (eobFlags[inRecIdx] == kStarchFalse) {
#ifdef __cplusplus
- STARCHCAT2_parseCoordinatesFromBedLineV2( &eobFlags[inRecIdx],
- reinterpret_cast<const char *>( extractedElements[inRecIdx] ),
- &starts[inRecIdx],
- &stops[inRecIdx]);
+ STARCHCAT2_parseCoordinatesFromBedLineV2p2( &eobFlags[inRecIdx],
+ reinterpret_cast<const char *>( extractedElements[inRecIdx] ),
+ &starts[inRecIdx],
+ &stops[inRecIdx],
+ &remainders[inRecIdx]);
#else
- STARCHCAT2_parseCoordinatesFromBedLineV2( &eobFlags[inRecIdx],
- (const char *) extractedElements[inRecIdx],
- &starts[inRecIdx],
- &stops[inRecIdx]);
+ STARCHCAT2_parseCoordinatesFromBedLineV2p2( &eobFlags[inRecIdx],
+ (const char *) extractedElements[inRecIdx],
+ &starts[inRecIdx],
+ &stops[inRecIdx],
+ &remainders[inRecIdx]);
#endif
+ }
}
/* identify the lowest element, put it into the compression buffer for later processing, and refill from the input stream, if needed */
+
#ifdef __cplusplus
- lowestElementRes = STARCHCAT2_identifyLowestBedElement( reinterpret_cast<const Boolean *>( eobFlags ),
- reinterpret_cast<const SignedCoordType *>( starts ),
- reinterpret_cast<const SignedCoordType *>( stops ),
- static_cast<const size_t>( summary->numRecords ),
- &lowestStartElementIdx);
+ lowestElementRes = STARCHCAT2_identifyLowestBedElementV2p2( reinterpret_cast<const Boolean *>( eobFlags ),
+ reinterpret_cast<const SignedCoordType *>( starts ),
+ reinterpret_cast<const SignedCoordType *>( stops ),
+ const_cast<const char **>( remainders ),
+ static_cast<const size_t>( summary->numRecords ),
+ &lowestStartElementIdx);
#else
- lowestElementRes = STARCHCAT2_identifyLowestBedElement( (const Boolean *) eobFlags,
- (const SignedCoordType *) starts,
- (const SignedCoordType *) stops,
- (const size_t) summary->numRecords,
- &lowestStartElementIdx);
+ lowestElementRes = STARCHCAT2_identifyLowestBedElementV2p2( (const Boolean *) eobFlags,
+ (const SignedCoordType *) starts,
+ (const SignedCoordType *) stops,
+ (const char **) remainders,
+ (const size_t) summary->numRecords,
+ &lowestStartElementIdx);
#endif
if ((eobFlags) && (starts) && (stops) && (lowestElementRes == STARCHCAT_EXIT_SUCCESS)) {
#ifdef __cplusplus
+ LineLengthType leLineMaxStringLength = static_cast<LineLengthType>(strlen(static_cast<const char *>( extractedElements[lowestStartElementIdx] )) - 1);
+ outputRetransformState->r_lineMaxStringLength = (outputRetransformState->r_lineMaxStringLength >= leLineMaxStringLength) ? outputRetransformState->r_lineMaxStringLength : leLineMaxStringLength;
STARCHCAT2_addLowestBedElementToCompressionBuffer( compressionBuffer,
- static_cast<const char *>( extractedElements[lowestStartElementIdx] ),
- &compressionLineCount);
+ static_cast<const char *>( extractedElements[lowestStartElementIdx] ),
+ &compressionLineCount);
#else
+ LineLengthType leLineMaxStringLength = (LineLengthType) strlen(extractedElements[lowestStartElementIdx]) - 1;
+ outputRetransformState->r_lineMaxStringLength = (outputRetransformState->r_lineMaxStringLength >= leLineMaxStringLength) ? outputRetransformState->r_lineMaxStringLength : leLineMaxStringLength;
STARCHCAT2_addLowestBedElementToCompressionBuffer( compressionBuffer,
- (const char *) extractedElements[lowestStartElementIdx],
- &compressionLineCount);
+ (const char *) extractedElements[lowestStartElementIdx],
+ &compressionLineCount);
#endif
/* 4 -- extract lowest element to extracted elements buffer */
STARCHCAT2_extractBedLine( &eobFlags[lowestStartElementIdx],
- extractionBuffers[lowestStartElementIdx],
- &extractionBufferOffsets[lowestStartElementIdx],
- &extractedElements[lowestStartElementIdx]);
+ extractionBuffers[lowestStartElementIdx],
+ &extractionBufferOffsets[lowestStartElementIdx],
+ &extractedElements[lowestStartElementIdx]);
+
extractedLineCounts[lowestStartElementIdx]--;
/*
@@ -2630,17 +2983,17 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
case kBzip2: {
if (STARCHCAT2_fillExtractionBufferFromBzip2Stream(&eofFlags[lowestStartElementIdx],
#ifdef __cplusplus
- const_cast<char *>( inChr ),
+ const_cast<char *>( inChr ),
#else
- (char *) inChr,
+ (char *) inChr,
#endif
- extractionBuffers[lowestStartElementIdx],
- &nExtractionBuffers[lowestStartElementIdx],
- &bzInFps[lowestStartElementIdx],
- &nBzReads[lowestStartElementIdx],
- extractionRemainderBufs[lowestStartElementIdx],
- &nExtractionRemainderBufs[lowestStartElementIdx],
- transformStates[lowestStartElementIdx]) != STARCHCAT_EXIT_SUCCESS) {
+ extractionBuffers[lowestStartElementIdx],
+ &nExtractionBuffers[lowestStartElementIdx],
+ &bzInFps[lowestStartElementIdx],
+ &nBzReads[lowestStartElementIdx],
+ extractionRemainderBufs[lowestStartElementIdx],
+ &nExtractionRemainderBufs[lowestStartElementIdx],
+ transformStates[lowestStartElementIdx]) != STARCHCAT_EXIT_SUCCESS) {
fprintf(stderr, "ERROR: Could not extract data from bzip2 input stream at index [%zu]!\n", lowestStartElementIdx);
return STARCHCAT_EXIT_FAILURE;
}
@@ -2648,19 +3001,19 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
}
case kGzip: {
if (STARCHCAT2_fillExtractionBufferFromGzipStream(&eofFlags[lowestStartElementIdx],
- &zInFps[lowestStartElementIdx],
+ &zInFps[lowestStartElementIdx],
#ifdef __cplusplus
- const_cast<char *>( inChr ),
+ const_cast<char *>( inChr ),
#else
- (char *) inChr,
+ (char *) inChr,
#endif
- extractionBuffers[lowestStartElementIdx],
- &nExtractionBuffers[lowestStartElementIdx],
- &zInStreams[lowestStartElementIdx],
- &nZReads[lowestStartElementIdx],
- &extractionRemainderBufs[lowestStartElementIdx],
- &nExtractionRemainderBufs[lowestStartElementIdx],
- transformStates[lowestStartElementIdx]) != STARCHCAT_EXIT_SUCCESS) {
+ extractionBuffers[lowestStartElementIdx],
+ &nExtractionBuffers[lowestStartElementIdx],
+ &zInStreams[lowestStartElementIdx],
+ &nZReads[lowestStartElementIdx],
+ &extractionRemainderBufs[lowestStartElementIdx],
+ &nExtractionRemainderBufs[lowestStartElementIdx],
+ transformStates[lowestStartElementIdx]) != STARCHCAT_EXIT_SUCCESS) {
fprintf(stderr, "ERROR: Could not extract data from gzip input stream at index [%zu]!\n", lowestStartElementIdx);
return STARCHCAT_EXIT_FAILURE;
}
@@ -2680,9 +3033,9 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
*/
extractionBufferOffsets[lowestStartElementIdx] = 0;
STARCHCAT2_extractBedLine(&eobFlags[lowestStartElementIdx],
- extractionBuffers[lowestStartElementIdx],
- &extractionBufferOffsets[lowestStartElementIdx],
- &extractedElements[lowestStartElementIdx]);
+ extractionBuffers[lowestStartElementIdx],
+ &extractionBufferOffsets[lowestStartElementIdx],
+ &extractedElements[lowestStartElementIdx]);
extractedLineCounts[lowestStartElementIdx]--;
}
}
@@ -2700,6 +3053,10 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
#ifdef DEBUG
fprintf(stderr, "STEP 5 - COMP BUFFER [%s] -->\nCOMPRESS RETRANSFORMATION BUFFER (FULL)\n[%s]\n", compressionBuffer, retransformedOutputBuffer);
#endif
+
+ /* hash the transformed buffer */
+ sha1_process_bytes (retransformedOutputBuffer, strlen(retransformedOutputBuffer), &r_perChromosomeHashCtx);
+
STARCHCAT2_resetCompressionBuffer(compressionBuffer, &compressionLineCount);
switch (outType) {
case kBzip2: {
@@ -2729,6 +3086,24 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
} while (allEOF == kStarchFalse);
+ /* finalize the hash of the transformed buffer */
+ sha1_finish_ctx (&r_perChromosomeHashCtx, r_sha1Digest);
+#ifdef __cplusplus
+ STARCH_encodeBase64(&r_base64EncodedSha1Digest,
+ static_cast<size_t>( STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH ),
+ reinterpret_cast<const unsigned char *>( r_sha1Digest ),
+ static_cast<size_t>( STARCH2_MD_FOOTER_SHA1_LENGTH ) );
+#else
+ STARCH_encodeBase64(&r_base64EncodedSha1Digest,
+ (const size_t) STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH,
+ (const unsigned char *) r_sha1Digest,
+ (const size_t) STARCH2_MD_FOOTER_SHA1_LENGTH);
+#endif
+ outputRetransformState->r_signature = r_base64EncodedSha1Digest;
+#ifdef DEBUG
+ fprintf(stderr, "\nPROGRESS: SHA-1 digest for chr [%s] is [%s]\n", inChr, r_base64EncodedSha1Digest);
+#endif
+
/* breakdown output stream */
switch (outType) {
case kBzip2: {
@@ -2776,6 +3151,8 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
finalTotalUniqueBases = outputRetransformState->r_totalUniqueBases;
finalDuplicateElementExists = outputRetransformState->r_duplicateElementExists;
finalNestedElementExists = outputRetransformState->r_nestedElementExists;
+ finalSignature = outputRetransformState->r_signature;
+ finalLineMaxStringLength = outputRetransformState->r_lineMaxStringLength;
STARCHCAT2_finalizeMetadata( outMd,
#ifdef __cplusplus
@@ -2789,7 +3166,9 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
finalTotalNonUniqueBases,
finalTotalUniqueBases,
finalDuplicateElementExists,
- finalNestedElementExists );
+ finalNestedElementExists,
+ finalSignature,
+ finalLineMaxStringLength );
/* breakdown */
for (inRecIdx = 0U; inRecIdx < summary->numRecords; inRecIdx++) {
@@ -2800,6 +3179,8 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
free(extractedElements[inRecIdx]), extractedElements[inRecIdx] = NULL;
if (extractionRemainderBufs[inRecIdx])
free(extractionRemainderBufs[inRecIdx]), extractionRemainderBufs[inRecIdx] = NULL;
+ if (remainders[inRecIdx])
+ free(remainders[inRecIdx]), remainders[inRecIdx] = NULL;
inRecord = *(summary->records) + inRecIdx;
inType = inRecord->type; /* get record type of input stream */
switch (inType) {
@@ -2846,6 +3227,8 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
free(starts), starts = NULL;
if (stops)
free(stops), stops = NULL;
+ if (remainders)
+ free(remainders), remainders = NULL;
if (transformStates)
free(transformStates), transformStates = NULL;
if (extractionRemainderBufs)
@@ -2866,6 +3249,8 @@ STARCHCAT2_mergeInputRecordsToOutput (const char *inChr, Metadata **outMd, const
free(outputRetransformState), outputRetransformState = NULL;
if (finalOutTagFn)
free(finalOutTagFn), finalOutTagFn = NULL;
+ if (r_base64EncodedSha1Digest)
+ free(r_base64EncodedSha1Digest), r_base64EncodedSha1Digest = NULL;
return STARCHCAT_EXIT_SUCCESS;
}
@@ -3115,7 +3500,7 @@ STARCHCAT_mergeInputRecordsToOutput (Metadata **outMd, const char *outTag, const
stops[inRecIdx] = (SignedCoordType) strtoll(fieldBuffer, NULL, 10);
else if (fieldIdx > 3)
break;
-#endif
+#endif
}
}
}
@@ -3460,12 +3845,12 @@ STARCHCAT_mergeChromosomeStreams (const ChromosomeSummaries *chrSums, const Comp
#endif
#ifdef __cplusplus
assert( STARCHCAT_mergeInputRecordsToOutput(&outputMd,
- reinterpret_cast<const char *>( outputTag ),
+ reinterpret_cast<const char *>( outputTag ),
static_cast<const CompressionType>( outputType ),
reinterpret_cast<const ChromosomeSummary *>( summary )) );
#else
assert( STARCHCAT_mergeInputRecordsToOutput(&outputMd,
- (const char *) outputTag,
+ (const char *) outputTag,
(const CompressionType) outputType,
(const ChromosomeSummary *) summary) );
#endif
@@ -3496,7 +3881,7 @@ STARCHCAT_mergeChromosomeStreams (const ChromosomeSummaries *chrSums, const Comp
static_cast<const Boolean>( hFlag ),
reinterpret_cast<const char *>( note )) );
assert( STARCH_mergeMetadataWithCompressedFiles( reinterpret_cast<const Metadata *>( headOutputMd ),
- dynamicMdBuffer) );
+ dynamicMdBuffer) );
assert( STARCH_deleteCompressedFiles( reinterpret_cast<const Metadata *>( headOutputMd )) );
#else
assert( STARCH_writeJSONMetadata( (const Metadata *) headOutputMd,
@@ -3505,7 +3890,7 @@ STARCHCAT_mergeChromosomeStreams (const ChromosomeSummaries *chrSums, const Comp
(const Boolean) hFlag,
(const char *) note) );
assert( STARCH_mergeMetadataWithCompressedFiles( (const Metadata *) headOutputMd,
- dynamicMdBuffer) );
+ dynamicMdBuffer) );
assert( STARCH_deleteCompressedFiles( (const Metadata *) headOutputMd) );
#endif
@@ -3657,14 +4042,14 @@ STARCHCAT_printChromosomeSummaries (const ChromosomeSummaries *chrSums)
"\t\t\t\tintSize: %" PRIu64 "\n" \
"\t\t\t\tintLineCount: %" PRId64 "\n", \
recIdx,
- mdRecord->filename,
- mdRecord->hFlag,
- static_cast<uint64_t>( mdRecord->mdOffset ),
- mdRecord->type,
- iter->chromosome,
- iter->filename,
- static_cast<uint64_t>( iter->size ),
- static_cast<uint64_t>( iter->lineCount ));
+ mdRecord->filename,
+ mdRecord->hFlag,
+ static_cast<uint64_t>( mdRecord->mdOffset ),
+ mdRecord->type,
+ iter->chromosome,
+ iter->filename,
+ static_cast<uint64_t>( iter->size ),
+ static_cast<uint64_t>( iter->lineCount ));
#else
fprintf(stderr, "\t\t\t[recIdx: %02u]\n" \
"\t\t\t\tfilename: %s\n" \
@@ -3676,14 +4061,14 @@ STARCHCAT_printChromosomeSummaries (const ChromosomeSummaries *chrSums)
"\t\t\t\tintSize: %" PRIu64 "\n" \
"\t\t\t\tintLineCount: %" PRId64 "\n", \
recIdx,
- mdRecord->filename,
- mdRecord->hFlag,
- (uint64_t) mdRecord->mdOffset,
- mdRecord->type,
- iter->chromosome,
- iter->filename,
- (uint64_t) iter->size,
- (uint64_t) iter->lineCount);
+ mdRecord->filename,
+ mdRecord->hFlag,
+ (uint64_t) mdRecord->mdOffset,
+ mdRecord->type,
+ iter->chromosome,
+ iter->filename,
+ (uint64_t) iter->size,
+ (uint64_t) iter->lineCount);
#endif
#endif
break;
@@ -4120,21 +4505,21 @@ STARCHCAT_buildMetadataRecords (json_t ***metadataJSONs, MetadataRecord **mdReco
/* parse the archive file's metadata */
if ( STARCH_readJSONMetadata( &mdJSON,
- &inFilePtr,
+ &inFilePtr,
#ifdef __cplusplus
- reinterpret_cast<const char *>( inFile ),
+ reinterpret_cast<const char *>( inFile ),
#else
- (const char *) inFile,
+ (const char *) inFile,
#endif
- &md,
- &inType,
- &version,
- &cTime,
- ¬e,
- &mdOffset,
- &hFlag,
- suppressErrorMsgs,
- preserveJSONRef) != STARCH_EXIT_SUCCESS ) {
+ &md,
+ &inType,
+ &version,
+ &cTime,
+ ¬e,
+ &mdOffset,
+ &hFlag,
+ suppressErrorMsgs,
+ preserveJSONRef) != STARCH_EXIT_SUCCESS ) {
fprintf(stderr, "ERROR: Could not read metadata from archive: %s (is this a starch file?)\n", inFile);
return STARCHCAT_EXIT_FAILURE;
}
@@ -4313,7 +4698,7 @@ STARCHCAT_printUsage (int errorType)
case STARCHCAT_FATAL_ERROR:
case STARCHCAT_HELP_ERROR:
default:
- fprintf(stderr, "%s\n citation: %s\n version: %s\n authors: %s\n%s\n", name, BEDOPS::citation(), BEDOPS::revision(), authors, usage);
+ fprintf(stderr, "%s\n citation: %s\n version: %s\n authors: %s\n%s\n", name, BEDOPS::citation(), BEDOPS::revision(), authors, usage);
break;
}
}
@@ -4398,6 +4783,23 @@ STARCHCAT_isArchiveConcurrentOrOlder (const ArchiveVersion *av)
}
Boolean
+STARCHCAT_isArchiveOlder (const ArchiveVersion *av)
+{
+#ifdef DEBUG
+ fprintf (stderr, "\n--- STARCHCAT_isArchiveOlder() ---\n");
+#endif
+
+ if ( ( (av->major < STARCH_MAJOR_VERSION) ) ||
+ ( (av->major == STARCH_MAJOR_VERSION) &&
+ (av->minor < STARCH_MINOR_VERSION) &&
+ (av->revision <= STARCH_REVISION_VERSION) )
+ )
+ return kStarchTrue;
+
+ return kStarchFalse;
+}
+
+Boolean
STARCHCAT_isArchiveNewer (const ArchiveVersion *av)
{
#ifdef DEBUG
@@ -4534,7 +4936,7 @@ STARCHCAT_checkMetadataJSONVersions (json_t ***mdJSONs, const unsigned int numRe
}
int
-STARCHCAT2_mergeChromosomeStreams (const ChromosomeSummaries *chrSums, const CompressionType outputType, const char *note, size_t *cumulativeOutputSize)
+STARCHCAT2_mergeChromosomeStreams (const ChromosomeSummaries *chrSums, const CompressionType outputType, const char *note, size_t *cumulativeOutputSize, const Boolean generatePerChrSignatureFlag, const Boolean reportProgressFlag, const LineCountType reportProgressN)
{
#ifdef DEBUG
fprintf(stderr, "\n--- STARCHCAT2_mergeChromosomeStreams() ---\n");
@@ -4551,7 +4953,7 @@ STARCHCAT2_mergeChromosomeStreams (const ChromosomeSummaries *chrSums, const Com
Boolean hFlag = kStarchFalse; /* starchcat does not currently support headers */
char *dynamicMdBuffer = NULL;
char *dynamicMdBufferCopy = NULL;
- unsigned char sha1Digest[STARCH2_MD_FOOTER_SHA1_LENGTH];
+ unsigned char sha1Digest[STARCH2_MD_FOOTER_SHA1_LENGTH] = {0};
char *base64EncodedSha1Digest = NULL;
char footerCumulativeRecordSizeBuffer[STARCH2_MD_FOOTER_CUMULATIVE_RECORD_SIZE_LENGTH + 1] = {0};
char footerRemainderBuffer[STARCH2_MD_FOOTER_REMAINDER_LENGTH + 1] = {0};
@@ -4641,47 +5043,94 @@ STARCHCAT2_mergeChromosomeStreams (const ChromosomeSummaries *chrSums, const Com
return STARCHCAT_EXIT_FAILURE;
}
+#ifdef DEBUG
+ fprintf(stderr, "kStarchTrue [%d]\n", (int) kStarchTrue);
+ fprintf(stderr, "STARCHCAT_isArchiveOlder [%d]\n", (int) STARCHCAT_isArchiveOlder((const ArchiveVersion *) inputRecord->av));
+ fprintf(stderr, "STARCHCAT_isArchiveConcurrent [%d]\n", (int) STARCHCAT_isArchiveConcurrent((const ArchiveVersion *) inputRecord->av));
+ fprintf(stderr, "STARCHCAT_isArchiveConcurrentOrOlder [%d]\n", (int) STARCHCAT_isArchiveConcurrentOrOlder((const ArchiveVersion *) inputRecord->av));
+ fprintf(stderr, "STARCHCAT_isArchiveNewerThan [%d]\n", (int) STARCHCAT_isArchiveNewerThan((const ArchiveVersion *) inputRecord->av, av120));
+ fprintf(stderr, "inputType [%d]\n", (int) inputType);
+ fprintf(stderr, "outputType [%d]\n", (int) outputType);
+#endif
+
#ifdef __cplusplus
if ( (inputType == outputType) &&
- (STARCHCAT_isArchiveConcurrentOrOlder(reinterpret_cast<const ArchiveVersion *>( inputRecord->av )) == kStarchTrue) &&
- (STARCHCAT_isArchiveNewerThan(reinterpret_cast<const ArchiveVersion *>( inputRecord->av ), av120)) ) {
+ (STARCHCAT_isArchiveConcurrent(reinterpret_cast<const ArchiveVersion *>( inputRecord->av )) == kStarchTrue) ) {
#else
if ( (inputType == outputType) &&
- (STARCHCAT_isArchiveConcurrentOrOlder((const ArchiveVersion *) inputRecord->av) == kStarchTrue) &&
- (STARCHCAT_isArchiveNewerThan((const ArchiveVersion *) inputRecord->av, av120)) ) {
+ (STARCHCAT_isArchiveConcurrent((const ArchiveVersion *) inputRecord->av) == kStarchTrue) ) {
#endif
#ifdef __cplusplus
assert( STARCHCAT2_copyInputRecordToOutput( &outputMd,
- reinterpret_cast<const char *>( outputTag ),
- static_cast<const CompressionType>( outputType ),
- reinterpret_cast<const char *>( inputChr ),
- reinterpret_cast<const MetadataRecord *>( inputRecord ),
- cumulativeOutputSize) );
+ reinterpret_cast<const char *>( outputTag ),
+ static_cast<const CompressionType>( outputType ),
+ reinterpret_cast<const char *>( inputChr ),
+ reinterpret_cast<const MetadataRecord *>( inputRecord ),
+ cumulativeOutputSize,
+ reportProgressFlag) );
#else
assert( STARCHCAT2_copyInputRecordToOutput( &outputMd,
- (const char *) outputTag,
- (const CompressionType) outputType,
- (const char *) inputChr,
- (const MetadataRecord *) inputRecord,
- cumulativeOutputSize) );
+ (const char *) outputTag,
+ (const CompressionType) outputType,
+ (const char *) inputChr,
+ (const MetadataRecord *) inputRecord,
+ cumulativeOutputSize,
+ reportProgressFlag) );
+#endif
+ }
+#ifdef __cplusplus
+ else if ( (inputType == outputType) &&
+ (STARCHCAT_isArchiveOlder(reinterpret_cast<const ArchiveVersion *>( inputRecord->av )) ) &&
+ (STARCHCAT_isArchiveNewerThan(reinterpret_cast<const ArchiveVersion *>( inputRecord->av ), av120)) ) {
+#else
+ else if ( (inputType == outputType) &&
+ (STARCHCAT_isArchiveOlder((const ArchiveVersion *) inputRecord->av)) &&
+ (STARCHCAT_isArchiveNewerThan((const ArchiveVersion *) inputRecord->av, av120)) ) {
+#endif
+#ifdef __cplusplus
+ assert( STARCHCAT2_rewriteInputRecordToOutput( &outputMd,
+ reinterpret_cast<const char *>( outputTag ),
+ static_cast<const CompressionType>( outputType ),
+ reinterpret_cast<const char *>( inputChr ),
+ reinterpret_cast<const MetadataRecord *>( inputRecord ),
+ cumulativeOutputSize,
+ generatePerChrSignatureFlag,
+ reportProgressFlag,
+ reportProgressN) );
+#else
+ assert( STARCHCAT2_rewriteInputRecordToOutput( &outputMd,
+ (const char *) outputTag,
+ (const CompressionType) outputType,
+ (const char *) inputChr,
+ (const MetadataRecord *) inputRecord,
+ cumulativeOutputSize,
+ generatePerChrSignatureFlag,
+ reportProgressFlag,
+ reportProgressN) );
#endif
}
else {
#ifdef __cplusplus
assert( STARCHCAT2_rewriteInputRecordToOutput( &outputMd,
- reinterpret_cast<const char *>( outputTag ),
- static_cast<const CompressionType>( outputType ),
- reinterpret_cast<const char *>( inputChr ),
- reinterpret_cast<const MetadataRecord *>( inputRecord ),
- cumulativeOutputSize) );
+ reinterpret_cast<const char *>( outputTag ),
+ static_cast<const CompressionType>( outputType ),
+ reinterpret_cast<const char *>( inputChr ),
+ reinterpret_cast<const MetadataRecord *>( inputRecord ),
+ cumulativeOutputSize,
+ generatePerChrSignatureFlag,
+ reportProgressFlag,
+ reportProgressN) );
#else
assert( STARCHCAT2_rewriteInputRecordToOutput( &outputMd,
- (const char *) outputTag,
- (const CompressionType) outputType,
- (const char *) inputChr,
- (const MetadataRecord *) inputRecord,
- cumulativeOutputSize) );
+ (const char *) outputTag,
+ (const CompressionType) outputType,
+ (const char *) inputChr,
+ (const MetadataRecord *) inputRecord,
+ cumulativeOutputSize,
+ generatePerChrSignatureFlag,
+ reportProgressFlag,
+ reportProgressN) );
#endif
}
if (!outputMd) {
@@ -4701,18 +5150,18 @@ STARCHCAT2_mergeChromosomeStreams (const ChromosomeSummaries *chrSums, const Com
else {
#ifdef __cplusplus
assert( STARCHCAT2_mergeInputRecordsToOutput( reinterpret_cast<const char *>( summary->chromosome ),
- &outputMd,
- reinterpret_cast<const char *>( outputTag ),
- static_cast<const CompressionType>( outputType ),
- reinterpret_cast<const ChromosomeSummary *>( summary ),
- cumulativeOutputSize) );
-#else
- assert( STARCHCAT2_mergeInputRecordsToOutput((const char *) summary->chromosome,
- &outputMd,
- (const char *) outputTag,
- (const CompressionType) outputType,
- (const ChromosomeSummary *) summary,
- cumulativeOutputSize) );
+ &outputMd,
+ reinterpret_cast<const char *>( outputTag ),
+ static_cast<const CompressionType>( outputType ),
+ reinterpret_cast<const ChromosomeSummary *>( summary ),
+ cumulativeOutputSize) );
+#else
+ assert( STARCHCAT2_mergeInputRecordsToOutput( (const char *) summary->chromosome,
+ &outputMd,
+ (const char *) outputTag,
+ (const CompressionType) outputType,
+ (const ChromosomeSummary *) summary,
+ cumulativeOutputSize) );
#endif
}
@@ -4735,17 +5184,17 @@ STARCHCAT2_mergeChromosomeStreams (const ChromosomeSummaries *chrSums, const Com
/* stitch up compressed files into one archive, along with metadata header */
#ifdef __cplusplus
- assert(STARCH_writeJSONMetadata(reinterpret_cast<const Metadata *>( headOutputMd ),
- &dynamicMdBuffer,
- const_cast<CompressionType *>( &outputType ),
- static_cast<const Boolean>( hFlag ),
- static_cast<const char *>( note )));
+ assert(STARCH_writeJSONMetadata( reinterpret_cast<const Metadata *>( headOutputMd ),
+ &dynamicMdBuffer,
+ const_cast<CompressionType *>( &outputType ),
+ static_cast<const Boolean>( hFlag ),
+ static_cast<const char *>( note )) );
#else
- assert(STARCH_writeJSONMetadata((const Metadata *) headOutputMd,
- &dynamicMdBuffer,
- (CompressionType *) &outputType,
- (const Boolean) hFlag,
- (const char *) note));
+ assert(STARCH_writeJSONMetadata( (const Metadata *) headOutputMd,
+ &dynamicMdBuffer,
+ (CompressionType *) &outputType,
+ (const Boolean) hFlag,
+ (const char *) note) );
#endif
fwrite(dynamicMdBuffer, 1, strlen(dynamicMdBuffer), stdout);
fflush(stdout);
@@ -4754,26 +5203,26 @@ STARCHCAT2_mergeChromosomeStreams (const ChromosomeSummaries *chrSums, const Com
dynamicMdBufferCopy = STARCH_strdup(dynamicMdBuffer);
#ifdef __cplusplus
STARCH_SHA1_All(reinterpret_cast<const unsigned char *>( dynamicMdBuffer ),
- strlen(dynamicMdBuffer),
- sha1Digest);
+ strlen(dynamicMdBuffer),
+ sha1Digest);
#else
STARCH_SHA1_All((const unsigned char *) dynamicMdBuffer,
- strlen(dynamicMdBuffer),
- sha1Digest);
+ strlen(dynamicMdBuffer),
+ sha1Digest);
#endif
free(dynamicMdBufferCopy);
/* encode signature in base64 encoding */
#ifdef __cplusplus
STARCH_encodeBase64(&base64EncodedSha1Digest,
- static_cast<const size_t>( STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH ),
- reinterpret_cast<const unsigned char *>( sha1Digest ),
- static_cast<const size_t>( STARCH2_MD_FOOTER_SHA1_LENGTH ));
+ static_cast<const size_t>( STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH ),
+ reinterpret_cast<const unsigned char *>( sha1Digest ),
+ static_cast<const size_t>( STARCH2_MD_FOOTER_SHA1_LENGTH ));
#else
STARCH_encodeBase64(&base64EncodedSha1Digest,
- (const size_t) STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH,
- (const unsigned char *) sha1Digest,
- (const size_t) STARCH2_MD_FOOTER_SHA1_LENGTH);
+ (const size_t) STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH,
+ (const unsigned char *) sha1Digest,
+ (const size_t) STARCH2_MD_FOOTER_SHA1_LENGTH);
#endif
/* build footer */
@@ -5325,16 +5774,16 @@ STARCHCAT2_fillExtractionBufferFromBzip2Stream (Boolean *eofFlag, char *recordCh
bzLineBuf,
tab,
t_startPtr,
- t_pLengthPtr,
- t_lastEndPtr,
+ t_pLengthPtr,
+ t_lastEndPtr,
t_firstInputToken,
- t_secondInputToken,
+ t_secondInputToken,
&t_currentChromosome,
- t_currentChromosomeLengthPtr,
- t_currentStartPtr,
- t_currentStopPtr,
+ t_currentChromosomeLengthPtr,
+ t_currentStartPtr,
+ t_currentStopPtr,
&t_currentRemainder,
- t_currentRemainderLengthPtr);
+ t_currentRemainderLengthPtr);
if (bzLineBuf[0] != 'p') {
(*t_lineIdxPtr)++;
@@ -5343,15 +5792,15 @@ STARCHCAT2_fillExtractionBufferFromBzip2Stream (Boolean *eofFlag, char *recordCh
#else
UNSTARCH_reverseTransformCoordinates( (const LineCountType) *t_lineIdxPtr,
#endif
- t_lastPositionPtr,
- t_lcDiffPtr,
- t_currentStartPtr,
- t_currentStopPtr,
- &t_currentRemainder,
- retransformedLineBuffer,
- &nRetransformedLineBuffer,
- &nRetransformedLineBufferPosition );
-
+ t_lastPositionPtr,
+ t_lcDiffPtr,
+ t_currentStartPtr,
+ t_currentStopPtr,
+ &t_currentRemainder,
+ retransformedLineBuffer,
+ &nRetransformedLineBuffer,
+ &nRetransformedLineBufferPosition );
+
/* resize the extraction buffer, if we're getting too close to the maximum size of a line */
#ifdef __cplusplus
if (static_cast<unsigned int>( *nExtractionBuffer - *t_nExtractionBufferPos ) < TOKENS_MAX_LENGTH) {
@@ -5375,30 +5824,30 @@ STARCHCAT2_fillExtractionBufferFromBzip2Stream (Boolean *eofFlag, char *recordCh
#ifdef __cplusplus
*t_nExtractionBuffer = (strlen(t_currentRemainder) > 0) ?
static_cast<size_t>( sprintf(extractionBuffer + *t_nExtractionBufferPos,
- "%s\t%" PRId64 "\t%" PRId64 "\t%s\n",
- t_currentChromosome,
- *t_currentStartPtr,
- *t_currentStopPtr,
- t_currentRemainder) )
- :
+ "%s\t%" PRId64 "\t%" PRId64 "\t%s\n",
+ t_currentChromosome,
+ *t_currentStartPtr,
+ *t_currentStopPtr,
+ t_currentRemainder) )
+ :
static_cast<size_t>( sprintf(extractionBuffer + *t_nExtractionBufferPos,
- "%s\t%" PRId64 "\t%" PRId64 "\n",
- t_currentChromosome,
- *t_currentStartPtr,
- *t_currentStopPtr) );
+ "%s\t%" PRId64 "\t%" PRId64 "\n",
+ t_currentChromosome,
+ *t_currentStartPtr,
+ *t_currentStopPtr) );
#else
*t_nExtractionBuffer = (strlen(t_currentRemainder) > 0) ?
(size_t) sprintf(extractionBuffer + *t_nExtractionBufferPos,
- "%s\t%" PRId64 "\t%" PRId64 "\t%s\n",
- t_currentChromosome,
- *t_currentStartPtr,
- *t_currentStopPtr,
- t_currentRemainder) :
+ "%s\t%" PRId64 "\t%" PRId64 "\t%s\n",
+ t_currentChromosome,
+ *t_currentStartPtr,
+ *t_currentStopPtr,
+ t_currentRemainder) :
(size_t) sprintf(extractionBuffer + *t_nExtractionBufferPos,
- "%s\t%" PRId64 "\t%" PRId64 "\n",
- t_currentChromosome,
- *t_currentStartPtr,
- *t_currentStopPtr);
+ "%s\t%" PRId64 "\t%" PRId64 "\n",
+ t_currentChromosome,
+ *t_currentStartPtr,
+ *t_currentStopPtr);
#endif
*t_nExtractionBufferPos += *t_nExtractionBuffer;
*(extractionBuffer + *t_nExtractionBufferPos) = '\0';
@@ -5623,19 +6072,19 @@ STARCHCAT2_fillExtractionBufferFromGzipStream (Boolean *eofFlag, FILE **inputFp,
fprintf(stderr, "\tzLineBuf -> [%s]\n", zLineBuf);
#endif
UNSTARCH_extractRawLine( recordChromosome,
- zLineBuf,
- tab,
- t_startPtr,
- t_pLengthPtr,
- t_lastEndPtr,
- t_firstInputToken,
- t_secondInputToken,
- &t_currentChromosome,
- t_currentChromosomeLengthPtr,
- t_currentStartPtr,
- t_currentStopPtr,
- &t_currentRemainder,
- t_currentRemainderLengthPtr);
+ zLineBuf,
+ tab,
+ t_startPtr,
+ t_pLengthPtr,
+ t_lastEndPtr,
+ t_firstInputToken,
+ t_secondInputToken,
+ &t_currentChromosome,
+ t_currentChromosomeLengthPtr,
+ t_currentStartPtr,
+ t_currentStopPtr,
+ &t_currentRemainder,
+ t_currentRemainderLengthPtr);
if (zLineBuf[0] != 'p') {
(*t_lineIdxPtr)++;
#ifdef __cplusplus
@@ -5643,14 +6092,14 @@ STARCHCAT2_fillExtractionBufferFromGzipStream (Boolean *eofFlag, FILE **inputFp,
#else
UNSTARCH_reverseTransformCoordinates( (const LineCountType) *t_lineIdxPtr,
#endif
- t_lastPositionPtr,
- t_lcDiffPtr,
- t_currentStartPtr,
- t_currentStopPtr,
- &t_currentRemainder,
- retransformedLineBuffer,
- &nRetransformedLineBuffer,
- &nRetransformedLineBufferPosition);
+ t_lastPositionPtr,
+ t_lcDiffPtr,
+ t_currentStartPtr,
+ t_currentStopPtr,
+ &t_currentRemainder,
+ retransformedLineBuffer,
+ &nRetransformedLineBuffer,
+ &nRetransformedLineBufferPosition);
/* resize the extraction buffer, if we're getting too close to the maximum size of a line */
if ((*nExtractionBuffer - *t_nExtractionBufferPos) < TOKENS_MAX_LENGTH) {
@@ -5677,31 +6126,31 @@ STARCHCAT2_fillExtractionBufferFromGzipStream (Boolean *eofFlag, FILE **inputFp,
#ifdef __cplusplus
*t_nExtractionBuffer = (t_currentRemainder) ?
static_cast<size_t>( sprintf(extractionBuffer + *t_nExtractionBufferPos,
- "%s\t%" PRId64 "\t%" PRId64 "\t%s\n",
- t_currentChromosome,
- *t_currentStartPtr,
- *t_currentStopPtr,
- t_currentRemainder) )
- :
+ "%s\t%" PRId64 "\t%" PRId64 "\t%s\n",
+ t_currentChromosome,
+ *t_currentStartPtr,
+ *t_currentStopPtr,
+ t_currentRemainder) )
+ :
static_cast<size_t>( sprintf(extractionBuffer + *t_nExtractionBufferPos,
- "%s\t%" PRId64 "\t%" PRId64 "\n",
- t_currentChromosome,
- *t_currentStartPtr,
- *t_currentStopPtr) );
+ "%s\t%" PRId64 "\t%" PRId64 "\n",
+ t_currentChromosome,
+ *t_currentStartPtr,
+ *t_currentStopPtr) );
#else
*t_nExtractionBuffer = (t_currentRemainder) ?
(size_t) sprintf(extractionBuffer + *t_nExtractionBufferPos,
- "%s\t%" PRId64 "\t%" PRId64 "\t%s\n",
- t_currentChromosome,
- *t_currentStartPtr,
- *t_currentStopPtr,
- t_currentRemainder)
- :
+ "%s\t%" PRId64 "\t%" PRId64 "\t%s\n",
+ t_currentChromosome,
+ *t_currentStartPtr,
+ *t_currentStopPtr,
+ t_currentRemainder)
+ :
(size_t) sprintf(extractionBuffer + *t_nExtractionBufferPos,
- "%s\t%" PRId64 "\t%" PRId64 "\n",
- t_currentChromosome,
- *t_currentStartPtr,
- *t_currentStopPtr);
+ "%s\t%" PRId64 "\t%" PRId64 "\n",
+ t_currentChromosome,
+ *t_currentStartPtr,
+ *t_currentStopPtr);
#endif
*t_nExtractionBufferPos += *t_nExtractionBuffer;
*(extractionBufferStart + *t_nExtractionBufferPos) = '\0';
@@ -5806,18 +6255,18 @@ STARCHCAT2_parseCoordinatesFromBedLineV2 (Boolean *eobFlag, const char *extracte
continue;
}
switch (fieldIdx) {
- case 1: {
- startStr[withinFieldIdx++] = extractedElement[charIdx];
- startStr[withinFieldIdx] = '\0';
- break;
- }
- case 2: {
- stopStr[withinFieldIdx++] = extractedElement[charIdx];
- stopStr[withinFieldIdx] = '\0';
- break;
- }
- default:
- break;
+ case 1: {
+ startStr[withinFieldIdx++] = extractedElement[charIdx];
+ startStr[withinFieldIdx] = '\0';
+ break;
+ }
+ case 2: {
+ stopStr[withinFieldIdx++] = extractedElement[charIdx];
+ stopStr[withinFieldIdx] = '\0';
+ break;
+ }
+ default:
+ break;
}
charIdx++;
}
@@ -5848,11 +6297,105 @@ STARCHCAT2_parseCoordinatesFromBedLineV2 (Boolean *eobFlag, const char *extracte
switch (errno) {
case EINVAL: {
+ fprintf(stderr, "ERROR: Result from parsing stop coordinate is not a valid number!\n");
+ return STARCH_EXIT_FAILURE;
+ }
+ case ERANGE: {
+ fprintf(stderr, "ERROR: Result from parsing stop coordinate is not within range of SignedCoordType (int64_t)!\n");
+ return STARCH_EXIT_FAILURE;
+ }
+ }
+ *stop = result;
+
+ return STARCH_EXIT_SUCCESS;
+}
+
+int
+STARCHCAT2_parseCoordinatesFromBedLineV2p2 (Boolean *eobFlag, const char *extractedElement, SignedCoordType *start, SignedCoordType *stop, char **remainder)
+{
+#ifdef DEBUG
+ fprintf(stderr, "\n--- STARCHCAT2_parseCoordinatesFromBedLineV2p2() ---\n");
+#endif
+
+ if (strlen(extractedElement) == 0) {
+#ifdef DEBUG
+ fprintf(stderr, "LEAVING EARLY\n");
+#endif
+ *eobFlag = kStarchTrue;
+ return STARCHCAT_EXIT_SUCCESS;
+ }
+
+ errno = 0;
+ int fieldIdx = 0;
+ int charIdx = 0;
+ int withinFieldIdx = 0;
+ static const char tab = '\t';
+ char startStr[MAX_DEC_INTEGERS + 1] = {0};
+ char stopStr[MAX_DEC_INTEGERS + 1] = {0};
+ SignedCoordType result = 0;
+
+ while (extractedElement[charIdx] != '\0') {
+ if (extractedElement[charIdx] == tab) {
+ if (fieldIdx < 3) {
+ withinFieldIdx = 0;
+ fieldIdx++;
+ }
+ charIdx++;
+ continue;
+ }
+ switch (fieldIdx) {
+ case 1: {
+ startStr[withinFieldIdx++] = extractedElement[charIdx];
+ startStr[withinFieldIdx] = '\0';
+ break;
+ }
+ case 2: {
+ stopStr[withinFieldIdx++] = extractedElement[charIdx];
+ stopStr[withinFieldIdx] = '\0';
+ break;
+ }
+ case 3: {
+ (*remainder)[withinFieldIdx++] = extractedElement[charIdx];
+ (*remainder)[withinFieldIdx] = '\0';
+ break;
+ }
+ default:
+ break;
+ }
+ charIdx++;
+ }
+
+#ifdef __cplusplus
+ result = static_cast<SignedCoordType>( strtoll(startStr, NULL, STARCH_RADIX) );
+#else
+ result = (SignedCoordType) strtoll(startStr, NULL, STARCH_RADIX);
+#endif
+
+ switch (errno) {
+ case EINVAL: {
fprintf(stderr, "ERROR: Result from parsing start coordinate is not a valid number!\n");
return STARCH_EXIT_FAILURE;
}
case ERANGE: {
- fprintf(stderr, "ERROR: Result from parsing start coordinate is not within range of SignedCoordType (uint64_t)!\n");
+ fprintf(stderr, "ERROR: Result from parsing start coordinate is not within range of SignedCoordType (int64_t)!\n");
+ return STARCH_EXIT_FAILURE;
+ }
+ }
+ *start = result;
+
+#ifdef __cplusplus
+ result = static_cast<SignedCoordType>( strtoll(stopStr, NULL, STARCH_RADIX) );
+#else
+ result = (SignedCoordType) strtoll(stopStr, NULL, STARCH_RADIX);
+#endif
+
+ switch (errno) {
+ case EINVAL: {
+ fprintf(stderr, "ERROR: Result from parsing stop coordinate is not a valid number!\n");
+ return STARCH_EXIT_FAILURE;
+ }
+ case ERANGE: {
+ fprintf(stderr, "ERROR: Result from parsing stop coordinate is not within range of SignedCoordType (int64_t)!\n");
return STARCH_EXIT_FAILURE;
}
}
@@ -6154,7 +6697,7 @@ STARCHCAT2_resetCompressionBuffer (char *compressionBuffer, LineCountType *compr
}
int
-STARCHCAT2_finalizeMetadata (Metadata **outMd, char *finalChromosome, char *finalOutTagFn, uint64_t finalStreamSize, LineCountType finalLineCount, uint64_t finalTotalNonUniqueBases, uint64_t finalTotalUniqueBases, Boolean finalDuplicateElementExists, Boolean finalNestedElementExists)
+STARCHCAT2_finalizeMetadata (Metadata **outMd, char *finalChromosome, char *finalOutTagFn, uint64_t finalStreamSize, LineCountType finalLineCount, uint64_t finalTotalNonUniqueBases, uint64_t finalTotalUniqueBases, Boolean finalDuplicateElementExists, Boolean finalNestedElementExists, char *finalSignature, LineLengthType finalLineMaxStringLength)
{
#ifdef DEBUG
fprintf(stderr, "\n--- STARCHCAT2_finalizeMetadata() ---\n");
@@ -6168,7 +6711,9 @@ STARCHCAT2_finalizeMetadata (Metadata **outMd, char *finalChromosome, char *fina
finalTotalNonUniqueBases,
finalTotalUniqueBases,
finalDuplicateElementExists,
- finalNestedElementExists );
+ finalNestedElementExists,
+ finalSignature,
+ finalLineMaxStringLength );
else
*outMd = STARCH_addMetadata( *outMd,
finalChromosome,
@@ -6178,7 +6723,9 @@ STARCHCAT2_finalizeMetadata (Metadata **outMd, char *finalChromosome, char *fina
finalTotalNonUniqueBases,
finalTotalUniqueBases,
finalDuplicateElementExists,
- finalNestedElementExists );
+ finalNestedElementExists,
+ finalSignature,
+ finalLineMaxStringLength );
return STARCH_EXIT_SUCCESS;
}
diff --git a/applications/bed/starch/src/starchcat.h b/applications/bed/starch/src/starchcat.h
index 6fedd38..3d1076e 100644
--- a/applications/bed/starch/src/starchcat.h
+++ b/applications/bed/starch/src/starchcat.h
@@ -6,7 +6,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -35,6 +35,7 @@
#include <getopt.h>
#include <bzlib.h>
#include <zlib.h>
+#include <errno.h>
#include "data/starch/unstarchHelpers.h"
#include "data/starch/starchMetadataHelpers.h"
@@ -111,6 +112,8 @@ typedef struct transformState {
uint64_t r_totalUniqueBases;
Boolean r_duplicateElementExists;
Boolean r_nestedElementExists;
+ char * r_signature;
+ LineLengthType r_lineMaxStringLength;
size_t r_nRetransBuf;
} TransformState;
@@ -135,39 +138,54 @@ typedef struct chromosomeSummaries {
static const char *name = "starchcat";
static const char *authors = "Alex Reynolds and Shane Neph";
static const char *usage = "\n" \
- "USAGE: starchcat [ --note=\"...\" ] [ --bzip2 | --gzip ] <starch-file-1> [<starch-file-2> ...]\n" \
+ "USAGE: starchcat [ --note=\"...\" ]\n" \
+ " [ --bzip2 | --gzip ]\n" \
+ " [ --omit-signature ]\n" \
+ " [ --report-progress=N ] <starch-file-1> [<starch-file-2> ...]\n" \
"\n" \
- " * At least one lexicographically-sorted, headerless starch archive is required.\n" \
- " While two or more inputs make sense for a multiset union operation, you can starchcat \n" \
- " one file in order to update its metadata, recompress it with a different backend method,\n" \
- " or add a note annotation.\n" \
+ " * At least one lexicographically-sorted, headerless starch archive is\n" \
+ " required.\n\n" \
+ " * While two or more inputs make sense for a multiset union operation, you\n" \
+ " can starchcat one file in order to update its metadata, recompress it\n" \
+ " with a different backend method, or add a note annotation.\n" \
"\n" \
- " * Compressed data are sent to standard output. Use the '>' operator to redirect\n" \
- " to a file.\n" \
+ " * Compressed data are sent to standard output. Use the '>' operator to\n" \
+ " redirect to a file.\n" \
"\n" \
- " Process Flags:\n\n" \
- " --note=\"foo bar...\" Append note to output archive metadata (optional)\n" \
- " --bzip2 | --gzip Specify backend compression type (optional, default is bzip2)\n" \
- " --version Show binary version\n" \
- " --help Show this usage message\n";
+ " Process Flags\n" \
+ " --------------------------------------------------------------------------\n" \
+ " --note=\"foo bar...\" Append note to output archive metadata (optional).\n\n" \
+ " --bzip2 | --gzip Specify backend compression type (optional, default\n" \
+ " is bzip2).\n\n" \
+ " --omit-signature Skip generating per-chromosome data integrity signature\n" \
+ " (optional, default is to generate signature).\n\n" \
+ " --report-progress=N Report compression progress every N elements per\n" \
+ " chromosome to standard error stream (optional)\n\n" \
+ " --version Show binary version.\n\n" \
+ " --help Show this usage message.\n";
static struct starchcat_client_global_args_t {
CompressionType compressionType;
char *note;
char **inputFiles;
size_t numberInputFiles;
+ Boolean generatePerChromosomeSignatureFlag;
+ Boolean reportProgressFlag;
+ LineCountType reportProgressN;
} starchcat_client_global_args;
static struct option starchcat_client_long_options[] = {
- {"note", required_argument, NULL, 'n'},
- {"bzip2", no_argument, NULL, 'b'},
- {"gzip", no_argument, NULL, 'g'},
- {"version", no_argument, NULL, 'v'},
- {"help", no_argument, NULL, 'h'},
- {NULL, no_argument, NULL, 0}
+ {"note", required_argument, NULL, 'n'},
+ {"bzip2", no_argument, NULL, 'b'},
+ {"gzip", no_argument, NULL, 'g'},
+ {"omit-signature", no_argument, NULL, 'o'},
+ {"report-progress", required_argument, NULL, 'r'},
+ {"version", no_argument, NULL, 'v'},
+ {"help", no_argument, NULL, 'h'},
+ {NULL, no_argument, NULL, 0 }
};
-static const char *starchcat_client_opt_string = "n:bgvh?";
+static const char *starchcat_client_opt_string = "n:bgorvh?";
void STARCHCAT_initializeGlobals();
@@ -179,7 +197,8 @@ int STARCHCAT2_copyInputRecordToOutput (Metadata **outMd,
const CompressionType outType,
const char *inChr,
const MetadataRecord *inRec,
- size_t *cumulativeOutputSize);
+ size_t *cumulativeOutputSize,
+ const Boolean reportProgressFlag);
int STARCHCAT_copyInputRecordToOutput (Metadata **outMd,
const char *outTag,
@@ -192,7 +211,10 @@ int STARCHCAT2_rewriteInputRecordToOutput (Metadata **outMd,
const CompressionType outType,
const char *inChr,
const MetadataRecord *inRec,
- size_t *cumulativeOutputSize);
+ size_t *cumulativeOutputSize,
+ const Boolean generatePerChrSignatureFlag,
+ const Boolean reportProgressFlag,
+ const LineCountType reportProgressN);
int STARCHCAT_rewriteInputRecordToOutput (Metadata **outMd,
const char *outTag,
@@ -211,6 +233,13 @@ int STARCHCAT2_identifyLowestBedElement(const Boolean *eobFlags,
const size_t numRecords,
size_t *lowestIdx);
+int STARCHCAT2_identifyLowestBedElementV2p2(const Boolean *eobFlags,
+ const SignedCoordType *starts,
+ const SignedCoordType *stops,
+ const char **remainders,
+ const size_t numRecords,
+ size_t *lowestIdx);
+
int STARCHCAT2_pullNextBedElement (const size_t recIdx,
const char **inLinesBuf,
const LineCountType *nInLinesBuf,
@@ -229,7 +258,10 @@ int STARCHCAT_mergeChromosomeStreams (const ChromosomeSummaries *chrSums,
int STARCHCAT2_mergeChromosomeStreams (const ChromosomeSummaries *chrSums,
const CompressionType outputType,
const char *note,
- size_t *cumulativeOutputSize);
+ size_t *cumulativeOutputSize,
+ const Boolean generatePerChrSignatureFlag,
+ const Boolean reportProgressFlag,
+ const LineCountType reportProgressN);
int STARCHCAT_freeChromosomeNames (char ***chrs,
unsigned int numChromosomes);
@@ -290,6 +322,8 @@ Boolean STARCHCAT_isArchiveConcurrent (const ArchiveVersion *av);
Boolean STARCHCAT_isArchiveConcurrentOrOlder (const ArchiveVersion *av);
+Boolean STARCHCAT_isArchiveOlder (const ArchiveVersion *av);
+
Boolean STARCHCAT_isArchiveNewer (const ArchiveVersion *av);
Boolean STARCHCAT_isArchiveNewerThan (const ArchiveVersion *av,
@@ -325,6 +359,7 @@ int STARCHCAT2_fillExtractionBufferFromBzip2Stream (Boolean *eofFlag, char
int STARCHCAT2_fillExtractionBufferFromGzipStream (Boolean *eofFlag, FILE **inputFp, char *recordChromosome, char *extractionBuffer, size_t *nExtractionBuffer, z_stream *zStream, size_t *nZRead, char **zRemainderBuf, size_t *nZRemainderBuf, TransformState *t_state);
int STARCHCAT2_extractBedLine (Boolean *eobFlag, char *extractionBuffer, int *extractionBufferOffset, char **extractedElement);
int STARCHCAT2_parseCoordinatesFromBedLineV2 (Boolean *eobFlag, const char *extractedElement, SignedCoordType *start, SignedCoordType *stop);
+int STARCHCAT2_parseCoordinatesFromBedLineV2p2 (Boolean *eobFlag, const char *extractedElement, SignedCoordType *start, SignedCoordType *stop, char **remainder);
int STARCHCAT2_addLowestBedElementToCompressionBuffer (char *compressionBuffer, const char *extractedElement, LineCountType *compressionLineCount);
int STARCHCAT2_transformCompressionBuffer (const char *compressionBuffer, char *retransformedOutputBuffer, TransformState *retransState);
int STARCHCAT2_squeezeRetransformedOutputBufferToBzip2Stream (BZFILE **bzStream, char *transformedBuffer);
@@ -339,7 +374,9 @@ int STARCHCAT2_finalizeMetadata (Metadata **outMd,
uint64_t finalTotalNonUniqueBases,
uint64_t finalTotalUniqueBases,
Boolean finalDuplicateElementExists,
- Boolean finalNestedElementExists);
+ Boolean finalNestedElementExists,
+ char *finalSignature,
+ LineLengthType finalLineMaxStringLength);
#ifdef __cplusplus
} // namespace starch
diff --git a/applications/bed/starch/src/starchcluster_gnuParallel.tcsh b/applications/bed/starch/src/starchcluster_gnuParallel.tcsh
index 6aae739..4e36f13 100755
--- a/applications/bed/starch/src/starchcluster_gnuParallel.tcsh
+++ b/applications/bed/starch/src/starchcluster_gnuParallel.tcsh
@@ -2,11 +2,11 @@
# author : sjn and apr
# date : Feb 2012
-# version : v2.5.0
+# version : v2.4.26
#
# BEDOPS
-# Copyright (C) 2011-2015 Shane Neph, Scott Kuehn and Alex Reynolds
+# Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
diff --git a/applications/bed/starch/src/starchcluster_sge.tcsh b/applications/bed/starch/src/starchcluster_sge.tcsh
index e8321f5..9ffb750 100755
--- a/applications/bed/starch/src/starchcluster_sge.tcsh
+++ b/applications/bed/starch/src/starchcluster_sge.tcsh
@@ -2,11 +2,11 @@
# author : sjn and apr
# date : Feb 2012
-# version : v2.5.0
+# version : v2.4.26
#
# BEDOPS
-# Copyright (C) 2011-2015 Shane Neph, Scott Kuehn and Alex Reynolds
+# Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
diff --git a/applications/bed/starch/src/starchcluster_sge.tcsh b/applications/bed/starch/src/starchcluster_slurm.tcsh
similarity index 82%
copy from applications/bed/starch/src/starchcluster_sge.tcsh
copy to applications/bed/starch/src/starchcluster_slurm.tcsh
index e8321f5..e8a955f 100755
--- a/applications/bed/starch/src/starchcluster_sge.tcsh
+++ b/applications/bed/starch/src/starchcluster_slurm.tcsh
@@ -1,12 +1,12 @@
#!/bin/tcsh -ef
# author : sjn and apr
-# date : Feb 2012
-# version : v2.5.0
+# date : 13 Sep 2016
+# version : v2.4.26
#
# BEDOPS
-# Copyright (C) 2011-2015 Shane Neph, Scott Kuehn and Alex Reynolds
+# Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -26,20 +26,15 @@
####################################################
# cluster variables:
# change to match your environment
-# may also require changes to 2 'qsub' calls below
####################################################
-set shell = "-S /bin/tcsh"
-set queue = "-q all.q"
-set misc_opts = "-V -cwd -w e -r yes -now no"
-set soundoff = "-j n -e /dev/null -o /dev/null"
-set sge_opts = "$queue $shell $misc_opts $soundoff"
+set slurm_opts = "-D $PWD -o /dev/null -e /dev/null"
############################
# some input error checking
############################
-set help = "\nUsage: starchcluster_sge [--help] [--clean] <input-bed-file> [output-starch-file]\n\n"
+set help = "\nUsage: starchcluster_slurm [--help] [--clean] <input-bed-file> [output-starch-file]\n\n"
set help = "$help Pass in the name of a BED file to create a starch archive using the cluster.\n\n"
set help = "$help (stdin isn't supported through this wrapper script, but starch supports it natively.)\n\n"
set help = "$help Add --clean to remove <input-bed-file> after starching it up.\n\n"
@@ -126,13 +121,10 @@ set files = ()
set jids = ()
@ cntr = 0
foreach chrom (`bedextract --list-chr $input`)
-
-qsub $sge_opts -N $nm.$cntr > /dev/stderr << __EXTRACTION__
- bedextract $chrom $input | starch - > $cntr
-__EXTRACTION__
-
- set jids = ($jids $nm.$cntr)
- set files = ($files $cntr)
+ set res = `sbatch $slurm_opts -J "$nm.$cntr" --wrap="module add bedops; bedextract $chrom $input | starch - > $nm/$cntr"`
+ set slurm_jid = `echo $res | sed 's|^Submitted batch job ||'`
+ set jids = ($jids $slurm_jid)
+ set files = ($files $nm/$cntr)
@ cntr++
end
@@ -145,14 +137,7 @@ endif
# create final starch archive and clean things up
##################################################
-qsub $sge_opts -N $nm.union -hold_jid `echo $jids | tr ' ' ','` > /dev/stderr << __CATTED__
- starchcat $files > $output
- cd $here
- rm -rf $nm
-
- if ( $clean > 0 ) then
- rm -f $originput
- endif
-__CATTED__
+set dependencies = `echo $jids | tr ' ' ':'`
+sbatch $slurm_opts -J $nm.union --dependency=afterok:$dependencies --wrap="module add bedops; starchcat $files > $output; cd $here; rm -rf $nm; if [ $clean != 0 ]; then rm -f $originput; fi;"
exit 0
diff --git a/applications/bed/starch/src/starchstrip.c b/applications/bed/starch/src/starchstrip.c
new file mode 100644
index 0000000..4650900
--- /dev/null
+++ b/applications/bed/starch/src/starchstrip.c
@@ -0,0 +1,898 @@
+//=========
+// Author: Alex Reynolds & Shane Neph
+// Project: starchstrip
+// File: starchstrip.c
+//=========
+
+//
+// BEDOPS
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+
+#include "starchstrip.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+
+#include "data/starch/starchHelpers.h"
+#include "data/starch/starchConstants.h"
+#include "data/starch/starchFileHelpers.h"
+#include "data/starch/starchBase64Coding.h"
+#include "data/starch/starchSha1Digest.h"
+#include "suite/BEDOPS.Version.hpp"
+
+#ifdef __cplusplus
+namespace {
+ using namespace starch;
+} // unnamed namespace
+#endif
+
+int
+main (int argc, char** argv)
+{
+#ifdef DEBUG
+ fprintf(stderr, "\n--- starchstrip main() - enter ---\n");
+#endif
+
+ setlocale(LC_ALL, "POSIX");
+ STARCHSTRIP_init_globals();
+ STARCHSTRIP_init_command_line_options(argc, argv);
+ STARCHSTRIP_init_chromosomes_list();
+ /* validate input archive */
+ STARCHSTRIP_init_archive_metadata();
+ STARCHSTRIP_check_archive_version();
+ /* validate input chromosome names against query names */
+ STARCHSTRIP_check_chromosome_stream_names();
+ /* write output archive */
+ STARCHSTRIP_write_header(stdout);
+ STARCHSTRIP_write_chromosome_streams(stdout);
+ STARCHSTRIP_write_updated_metadata(stdout);
+ /* cleanup */
+ STARCHSTRIP_delete_globals();
+
+#ifdef DEBUG
+ fprintf(stderr, "\n--- starchstrip main() - exit ---\n");
+#endif
+
+ return EXIT_SUCCESS;
+}
+
+#ifdef __cplusplus
+namespace starch {
+#endif
+
+static void
+STARCHSTRIP_write_updated_metadata(FILE* os)
+{
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_write_updated_metadata() - enter ---\n");
+#endif
+
+ char *md_json_buffer = NULL;
+ unsigned char sha1_digest[STARCH2_MD_FOOTER_SHA1_LENGTH] = {0};
+ char *base64_encoded_sha1_digest = NULL;
+ char footer_cumulative_record_size_buffer[STARCH2_MD_FOOTER_CUMULATIVE_RECORD_SIZE_LENGTH + 1] = {0};
+ char footer_remainder_buffer[STARCH2_MD_FOOTER_REMAINDER_LENGTH + 1] = {0};
+ char footer_buffer[STARCH2_MD_FOOTER_LENGTH + 1] = {0};
+
+ if (!starchstrip_globals.output_records) {
+ fprintf(stderr, "Error: Output metadata structure is empty after stripping records from input -- something went wrong in mid-stream\n");
+ exit(EIO);
+ }
+
+ /* archive version and creation timestamp are NULL, in order to write default values */
+ md_json_buffer = STARCH_generateJSONMetadata(starchstrip_globals.output_records,
+ starchstrip_globals.archive_type,
+ starchstrip_globals.archive_version,
+ NULL,
+ starchstrip_globals.archive_note,
+ starchstrip_globals.archive_header_flag);
+ if (!md_json_buffer) {
+ fprintf(stderr, "Error: Could not write JSON-formatted metadata to buffer\n");
+ exit(EIO);
+ }
+ fwrite(md_json_buffer, 1, strlen(md_json_buffer), os);
+ fflush(os);
+
+#ifdef __cplusplus
+ STARCH_SHA1_All(reinterpret_cast<const unsigned char *>( md_json_buffer ), strlen(md_json_buffer), sha1_digest);
+ STARCH_encodeBase64(&base64_encoded_sha1_digest,
+ static_cast<const size_t>( STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH ),
+ reinterpret_cast<const unsigned char *>( sha1_digest ),
+ static_cast<const size_t>( STARCH2_MD_FOOTER_SHA1_LENGTH ));
+ sprintf(footer_cumulative_record_size_buffer, "%020llu", static_cast<unsigned long long>( starchstrip_globals.cumulative_output_size ));
+#else
+ STARCH_SHA1_All((const unsigned char *) md_json_buffer, strlen(md_json_buffer), sha1_digest);
+ STARCH_encodeBase64(&base64_encoded_sha1_digest,
+ (const size_t) STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH,
+ (const unsigned char *) sha1_digest,
+ (const size_t) STARCH2_MD_FOOTER_SHA1_LENGTH);
+ sprintf(footer_cumulative_record_size_buffer, "%020llu", (unsigned long long) starchstrip_globals.cumulative_output_size);
+#endif
+
+ memcpy(footer_buffer, footer_cumulative_record_size_buffer, strlen(footer_cumulative_record_size_buffer));
+ memcpy(footer_buffer + STARCH2_MD_FOOTER_CUMULATIVE_RECORD_SIZE_LENGTH, base64_encoded_sha1_digest, STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH - 1); /* strip trailing null */
+#ifdef __cplusplus
+ memset(footer_remainder_buffer, STARCH2_MD_FOOTER_REMAINDER_UNUSED_CHAR, static_cast<size_t>( STARCH2_MD_FOOTER_REMAINDER_LENGTH ));
+#else
+ memset(footer_remainder_buffer, STARCH2_MD_FOOTER_REMAINDER_UNUSED_CHAR, (size_t) STARCH2_MD_FOOTER_REMAINDER_LENGTH);
+#endif
+
+ memcpy(footer_buffer + STARCH2_MD_FOOTER_CUMULATIVE_RECORD_SIZE_LENGTH + STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH - 1, footer_remainder_buffer, STARCH2_MD_FOOTER_REMAINDER_LENGTH); /* don't forget to offset pointer index by -1 for base64-sha1's null */
+ footer_buffer[STARCH2_MD_FOOTER_CUMULATIVE_RECORD_SIZE_LENGTH + STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH - 1 + STARCH2_MD_FOOTER_REMAINDER_LENGTH - 1] = '\0';
+ footer_buffer[STARCH2_MD_FOOTER_CUMULATIVE_RECORD_SIZE_LENGTH + STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH - 1 + STARCH2_MD_FOOTER_REMAINDER_LENGTH - 2] = '\n';
+ fprintf(os, "%s", footer_buffer);
+ fflush(os);
+
+ /* cleanup */
+ free(md_json_buffer), md_json_buffer = NULL;
+ free(base64_encoded_sha1_digest), base64_encoded_sha1_digest = NULL;
+
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_write_updated_metadata() - exit ---\n");
+#endif
+}
+
+static void
+STARCHSTRIP_write_chromosome_streams(FILE* os)
+{
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_write_chromosome_streams() - enter ---\n");
+#endif
+
+ Metadata* iter = NULL;
+ Metadata* output_records_tail = NULL;
+ size_t chr_to_process_idx = 0;
+ uint64_t start_offset = 0;
+ uint64_t bytes_to_copy = 0;
+ size_t bytes_read = 0;
+ char byte_buffer[starchstrip_copy_buffer_size];
+ int records_added = 0;
+
+ if (starchstrip_globals.archive_version->major == 2) {
+ start_offset += STARCH2_MD_HEADER_BYTE_LENGTH;
+ for (iter = starchstrip_globals.archive_records; iter != NULL; iter = iter->next) {
+ if ((starchstrip_globals.exclusion_flag) && (strcmp(iter->chromosome, starchstrip_globals.chromosomes_to_process[chr_to_process_idx]) == 0)) {
+ continue;
+ }
+ if (((starchstrip_globals.inclusion_flag) && (strcmp(iter->chromosome, starchstrip_globals.chromosomes_to_process[chr_to_process_idx]) == 0)) || ((starchstrip_globals.exclusion_flag) && (strcmp(iter->chromosome, starchstrip_globals.chromosomes_to_process[chr_to_process_idx]) != 0)) ) {
+#ifdef __cplusplus
+ fseeko(starchstrip_globals.archive_fp, static_cast<off_t>( start_offset ), SEEK_SET);
+#else
+ fseeko(starchstrip_globals.archive_fp, (off_t) start_offset, SEEK_SET);
+#endif
+ // copy
+ bytes_to_copy = iter->size;
+ do {
+ if (bytes_to_copy > starchstrip_copy_buffer_size) {
+ bytes_read = fread(byte_buffer, sizeof(char), starchstrip_copy_buffer_size, starchstrip_globals.archive_fp);
+ if (bytes_read != starchstrip_copy_buffer_size) {
+ fprintf(stderr, "Error: Could not copy 'starchstrip_copy_buffer_size' bytes into intermediate buffer\n");
+ exit(EIO); /* Input/output error (POSIX.1) */
+ }
+ fwrite(byte_buffer, sizeof(char), starchstrip_copy_buffer_size, os);
+ bytes_to_copy -= starchstrip_copy_buffer_size;
+ }
+ else {
+#ifdef __cplusplus
+ bytes_read = fread(byte_buffer, sizeof(char), static_cast<size_t>( bytes_to_copy ), starchstrip_globals.archive_fp);
+#else
+ bytes_read = fread(byte_buffer, sizeof(char), (size_t) bytes_to_copy, starchstrip_globals.archive_fp);
+#endif
+ if (bytes_read != bytes_to_copy) {
+ fprintf(stderr, "Error: Could not copy 'bytes_to_copy' bytes into intermediate buffer\n");
+ exit(EIO); /* Input/output error (POSIX.1) */
+ }
+#ifdef __cplusplus
+ fwrite(byte_buffer, sizeof(char), static_cast<size_t>( bytes_to_copy ), os);
+#else
+ fwrite(byte_buffer, sizeof(char), (size_t) bytes_to_copy, os);
+#endif
+ bytes_to_copy = 0;
+ }
+ } while (bytes_to_copy > 0);
+
+ // increment cumulative output file size
+ starchstrip_globals.cumulative_output_size += iter->size;
+
+#ifdef __cplusplus
+ if (records_added == 0) {
+ output_records_tail = STARCH_createMetadata( const_cast<char *>( iter->chromosome ),
+ iter->filename,
+ iter->size,
+ iter->lineCount,
+ iter->totalNonUniqueBases,
+ iter->totalUniqueBases,
+ iter->duplicateElementExists,
+ iter->nestedElementExists,
+ iter->signature,
+ iter->lineMaxStringLength );
+ starchstrip_globals.output_records = output_records_tail;
+ }
+ else {
+ output_records_tail = STARCH_addMetadata( output_records_tail,
+ const_cast<char *>( iter->chromosome ),
+ iter->filename,
+ iter->size,
+ iter->lineCount,
+ iter->totalNonUniqueBases,
+ iter->totalUniqueBases,
+ iter->duplicateElementExists,
+ iter->nestedElementExists,
+ iter->signature,
+ iter->lineMaxStringLength );
+ }
+#else
+ if (records_added == 0) {
+ output_records_tail = STARCH_createMetadata( (char *) iter->chromosome,
+ iter->filename,
+ iter->size,
+ iter->lineCount,
+ iter->totalNonUniqueBases,
+ iter->totalUniqueBases,
+ iter->duplicateElementExists,
+ iter->nestedElementExists,
+ iter->signature,
+ iter->lineMaxStringLength );
+ starchstrip_globals.output_records = output_records_tail;
+ }
+ else {
+ output_records_tail = STARCH_addMetadata( output_records_tail,
+ (char *) iter->chromosome,
+ iter->filename,
+ iter->size,
+ iter->lineCount,
+ iter->totalNonUniqueBases,
+ iter->totalUniqueBases,
+ iter->duplicateElementExists,
+ iter->nestedElementExists,
+ iter->signature,
+ iter->lineMaxStringLength );
+ }
+#endif
+ // increment records
+ records_added++;
+
+ // increment query chromosome within bounds
+ if (chr_to_process_idx < (starchstrip_globals.chromosomes_to_process_num - 1)) {
+ chr_to_process_idx++;
+ }
+ }
+ start_offset += iter->size;
+ }
+ }
+
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_write_chromosome_streams() - exit ---\n");
+#endif
+}
+
+static void
+STARCHSTRIP_write_header(FILE* os)
+{
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_write_header() - enter ---\n");
+#endif
+
+ if (starchstrip_globals.archive_version->major == 2) {
+ unsigned char* archive_header = NULL;
+ if (STARCH2_initializeStarchHeader(&archive_header) != STARCH_EXIT_SUCCESS) {
+ fprintf(stderr, "Error: Could not initialize output archive header\n");
+ exit(EIO); /* Input/output error (POSIX.1) */
+ }
+ if (STARCH2_writeStarchHeaderToOutputFp(archive_header, os) != STARCH_EXIT_SUCCESS) {
+ fprintf(stderr, "Error: Could not write archive header to output file stream\n");
+ exit(EIO); /* Input/output error (POSIX.1) */
+ }
+ free(archive_header), archive_header = NULL;
+ starchstrip_globals.cumulative_output_size += STARCH2_MD_HEADER_BYTE_LENGTH;
+ }
+ else {
+ fprintf(stderr, "Error: Unable to write archive header (archive version unsupported)\n");
+ exit(EINVAL); /* Invalid argument (POSIX.1) */
+ }
+
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_write_header() - exit ---\n");
+#endif
+}
+
+static void
+STARCHSTRIP_check_chromosome_stream_names()
+{
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_check_chromosome_stream_names() - enter ---\n");
+#endif
+
+ Metadata* iter = NULL;
+ size_t chr_idx = 0;
+ size_t chr_to_process_idx = 0;
+ size_t num_records = 0;
+ size_t num_records_to_include = 0;
+ size_t num_records_to_exclude = 0;
+
+ // loop through input metadata records, compare against chromosome name array
+ if (starchstrip_globals.archive_version->major == 2) {
+ // full count
+ for (iter = starchstrip_globals.archive_records; iter != NULL; iter = iter->next) {
+ num_records++;
+ }
+ // inclusion
+ if (starchstrip_globals.inclusion_flag) {
+ for (iter = starchstrip_globals.archive_records; iter != NULL; iter = iter->next) {
+ while (strcmp(iter->chromosome, starchstrip_globals.chromosomes[chr_idx]) > 0) {
+ chr_idx++;
+ if (chr_idx == starchstrip_globals.chromosomes_num) {
+ break;
+ }
+ }
+ if (chr_idx == starchstrip_globals.chromosomes_num) {
+ break;
+ }
+ if (strcmp(iter->chromosome, starchstrip_globals.chromosomes[chr_idx]) == 0) {
+ num_records_to_include++;
+ //fprintf(stderr, "Debug: Including [%s]\n", starchstrip_globals.chromosomes[chr_idx]);
+ chr_idx++;
+ if (chr_idx == starchstrip_globals.chromosomes_num) {
+ break;
+ }
+ }
+ }
+ //fprintf(stderr, "Debug: Including [%d] of [%d] records\n", num_records_to_include, num_records);
+ if (num_records_to_include == 0) {
+ fprintf(stderr, "Error: No chromosomes were found in input archive with matching names (output would be empty)\n");
+ exit(EINVAL); /* Invalid argument (POSIX.1) */
+ }
+ if (num_records_to_include == num_records) {
+ fprintf(stderr, "Error: All specified chromosome names were found in input archive (output would be identical to input)\n");
+ exit(EINVAL); /* Invalid argument (POSIX.1) */
+ }
+ starchstrip_globals.chromosomes_to_process_num = num_records_to_include;
+#ifdef __cplusplus
+ starchstrip_globals.chromosomes_to_process = static_cast<char**>( malloc(sizeof(char*) * starchstrip_globals.chromosomes_to_process_num) );
+#else
+ starchstrip_globals.chromosomes_to_process = malloc(sizeof(char*) * starchstrip_globals.chromosomes_to_process_num);
+#endif
+ for (chr_to_process_idx = 0; chr_to_process_idx < starchstrip_globals.chromosomes_to_process_num; ++chr_to_process_idx) {
+ starchstrip_globals.chromosomes_to_process[chr_to_process_idx] = NULL;
+ }
+ chr_idx = 0;
+ chr_to_process_idx = 0;
+ for (iter = starchstrip_globals.archive_records; iter != NULL; iter = iter->next) {
+ while (strcmp(iter->chromosome, starchstrip_globals.chromosomes[chr_idx]) > 0) {
+ chr_idx++;
+ if (chr_idx == starchstrip_globals.chromosomes_num) {
+ break;
+ }
+ }
+ if (chr_idx == starchstrip_globals.chromosomes_num) {
+ break;
+ }
+ if (strcmp(iter->chromosome, starchstrip_globals.chromosomes[chr_idx]) == 0) {
+#ifdef __cplusplus
+ starchstrip_globals.chromosomes_to_process[chr_to_process_idx] = static_cast <char*>( malloc( strlen(iter->chromosome) + 1 ) );
+#else
+ starchstrip_globals.chromosomes_to_process[chr_to_process_idx] = malloc(strlen(iter->chromosome) + 1);
+#endif
+ if (!starchstrip_globals.chromosomes_to_process[chr_to_process_idx]) {
+ fprintf(stderr, "Error: Could not allocate space for chromosome-to-process list argument\n");
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ memcpy(starchstrip_globals.chromosomes_to_process[chr_to_process_idx], iter->chromosome, strlen(iter->chromosome) + 1);
+ chr_to_process_idx++;
+ chr_idx++;
+ if (chr_idx == starchstrip_globals.chromosomes_num) {
+ break;
+ }
+ }
+ }
+ }
+
+ // exclusion
+ else if (starchstrip_globals.exclusion_flag) {
+ for (iter = starchstrip_globals.archive_records; iter != NULL; iter = iter->next) {
+ while (strcmp(iter->chromosome, starchstrip_globals.chromosomes[chr_idx]) > 0) {
+ chr_idx++;
+ if (chr_idx == starchstrip_globals.chromosomes_num) {
+ break;
+ }
+ }
+ if (chr_idx == starchstrip_globals.chromosomes_num) {
+ break;
+ }
+ if (strcmp(iter->chromosome, starchstrip_globals.chromosomes[chr_idx]) == 0) {
+ num_records_to_exclude++;
+ //fprintf(stderr, "Debug: Excluding [%s]\n", starchstrip_globals.chromosomes[chr_idx]);
+ chr_idx++;
+ if (chr_idx == starchstrip_globals.chromosomes_num) {
+ break;
+ }
+ }
+ }
+ //fprintf(stderr, "Debug: Excluding [%d] of [%d] records\n", num_records_to_exclude, num_records);
+ if (num_records_to_exclude == 0) {
+ fprintf(stderr, "Error: No chromosomes were found in input archive with matching names (output would be identical to input)\n");
+ exit(EINVAL); /* Invalid argument (POSIX.1) */
+ }
+ if (num_records_to_exclude == num_records) {
+ fprintf(stderr, "Error: All specified chromosome names were found in input archive (output would be empty)\n");
+ exit(EINVAL); /* Invalid argument (POSIX.1) */
+ }
+ starchstrip_globals.chromosomes_to_process_num = num_records_to_exclude;
+#ifdef __cplusplus
+ starchstrip_globals.chromosomes_to_process = static_cast<char**>( malloc(sizeof(char*) * starchstrip_globals.chromosomes_to_process_num) );
+#else
+ starchstrip_globals.chromosomes_to_process = malloc(sizeof(char*) * starchstrip_globals.chromosomes_to_process_num);
+#endif
+ for (chr_idx = 0; chr_idx < starchstrip_globals.chromosomes_to_process_num; ++chr_idx) {
+ starchstrip_globals.chromosomes_to_process[chr_idx] = NULL;
+ }
+ chr_idx = 0;
+ chr_to_process_idx = 0;
+ for (iter = starchstrip_globals.archive_records; iter != NULL; iter = iter->next) {
+ while (strcmp(iter->chromosome, starchstrip_globals.chromosomes[chr_idx]) > 0) {
+ chr_idx++;
+ if (chr_idx == starchstrip_globals.chromosomes_num) {
+ break;
+ }
+ }
+ if (chr_idx == starchstrip_globals.chromosomes_num) {
+ break;
+ }
+ if (strcmp(iter->chromosome, starchstrip_globals.chromosomes[chr_idx]) == 0) {
+#ifdef __cplusplus
+ starchstrip_globals.chromosomes_to_process[chr_to_process_idx] = static_cast <char*>( malloc( strlen(iter->chromosome) + 1 ) );
+#else
+ starchstrip_globals.chromosomes_to_process[chr_to_process_idx] = malloc(strlen(iter->chromosome) + 1);
+#endif
+ if (!starchstrip_globals.chromosomes_to_process[chr_to_process_idx]) {
+ fprintf(stderr, "Error: Could not allocate space for chromosome-to-process list argument\n");
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ memcpy(starchstrip_globals.chromosomes_to_process[chr_to_process_idx], iter->chromosome, strlen(iter->chromosome) + 1);
+ chr_to_process_idx++;
+ chr_idx++;
+ if (chr_idx == starchstrip_globals.chromosomes_num) {
+ break;
+ }
+ }
+ }
+ }
+ }
+
+
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_check_chromosome_stream_names() - exit ---\n");
+#endif
+}
+
+static void
+STARCHSTRIP_check_archive_version()
+{
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_check_archive_version() - enter ---\n");
+#endif
+
+ if ((starchstrip_globals.archive_version->major < starchstrip_archive_version_major_minimum)
+ ||
+ ((starchstrip_globals.archive_version->major == starchstrip_archive_version_major_minimum) && (starchstrip_globals.archive_version->minor < starchstrip_archive_version_minor_minimum))) {
+ fprintf(stderr, "Error: Archive must be v2.1 or greater -- use 'starchcat' to update archive, if needed\n");
+ exit(EINVAL); /* Invalid argument (POSIX.1) */
+ }
+
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_check_archive_version() - exit ---\n");
+#endif
+}
+
+static void
+STARCHSTRIP_init_archive_metadata()
+{
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_init_archive_metadata() - enter ---\n");
+#endif
+
+ if (STARCH_readJSONMetadata( &starchstrip_globals.archive_metadata_json,
+ &starchstrip_globals.archive_fp,
+#ifdef __cplusplus
+ reinterpret_cast<const char *>( starchstrip_globals.archive_fn ),
+#else
+ (const char *) starchstrip_globals.archive_fn,
+#endif
+ &starchstrip_globals.archive_records,
+ &starchstrip_globals.archive_type,
+ &starchstrip_globals.archive_version,
+ &starchstrip_globals.archive_timestamp,
+ &starchstrip_globals.archive_note,
+ &starchstrip_globals.archive_metadata_offset,
+ &starchstrip_globals.archive_header_flag,
+ starchstrip_globals.archive_suppress_error_msgs,
+ starchstrip_globals.archive_preserve_json_ref) != STARCH_EXIT_SUCCESS) {
+ fprintf(stderr, "Error: Could not read metadata from archive -- use 'unstarch --is-starch' to test if archive is valid\n");
+ exit(EINVAL); /* Invalid argument (POSIX.1) */
+ }
+
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_init_archive_metadata() - exit ---\n");
+#endif
+}
+
+/*
+ specifying special attribute for STARCHSTRIP_debug_chromosomes_to_query_list() to avoid: "warning: unused
+ function 'STARCHSTRIP_debug_chromosomes_to_query_list' [-Wunused-function]" message during non-debug compilation
+ cf. http://gcc.gnu.org/onlinedocs/gcc-3.4.1/gcc/Function-Attributes.html#Function%20Attributes
+*/
+#if defined(__GNUC__)
+static void STARCHSTRIP_debug_chromosomes_to_query_list() __attribute__ ((unused));
+#endif
+
+static void
+STARCHSTRIP_debug_chromosomes_to_query_list()
+{
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_debug_chromosomes_to_query_list() - enter ---\n");
+#endif
+
+ fprintf(stderr, "number of chromosomes to query [%zu]\n", starchstrip_globals.chromosomes_num);
+ for (size_t chr_idx = 0; chr_idx < starchstrip_globals.chromosomes_num; ++chr_idx) {
+ fprintf(stderr, "chr [%s]\n", starchstrip_globals.chromosomes[chr_idx]);
+ }
+
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_debug_chromosomes_to_query_list() - exit ---\n");
+#endif
+}
+
+/*
+ specifying special attribute for STARCHSTRIP_debug_chromosomes_to_process_list() to avoid: "warning: unused
+ function 'STARCHSTRIP_debug_chromosomes_to_process_list' [-Wunused-function]" message during non-debug compilation
+ cf. http://gcc.gnu.org/onlinedocs/gcc-3.4.1/gcc/Function-Attributes.html#Function%20Attributes
+*/
+#if defined(__GNUC__)
+static void STARCHSTRIP_debug_chromosomes_to_process_list() __attribute__ ((unused));
+#endif
+
+static void
+STARCHSTRIP_debug_chromosomes_to_process_list()
+{
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_debug_chromosomes_to_process_list() - enter ---\n");
+#endif
+
+ fprintf(stderr, "number of chromosomes to process [%zu]\n", starchstrip_globals.chromosomes_to_process_num);
+ for (size_t chr_to_process_idx = 0; chr_to_process_idx < starchstrip_globals.chromosomes_to_process_num; ++chr_to_process_idx) {
+ fprintf(stderr, "chr [%s]\n", starchstrip_globals.chromosomes_to_process[chr_to_process_idx]);
+ }
+
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_debug_chromosomes_to_process_list() - exit ---\n");
+#endif
+}
+
+static void
+STARCHSTRIP_init_chromosomes_list()
+{
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_init_chromosomes_list() - enter ---\n");
+#endif
+
+ /* count delimiters */
+ int delimiters_found = 0;
+ int pos = 0;
+ while (starchstrip_globals.chromosomes_str[pos] != '\0') {
+ if (starchstrip_globals.chromosomes_str[pos++] == starchstrip_chromosomes_str_delimiter) {
+ delimiters_found++;
+ }
+ }
+#ifdef __cplusplus
+ starchstrip_globals.chromosomes_num = static_cast<size_t>( delimiters_found + 1 );
+ starchstrip_globals.chromosomes = static_cast<char**>( malloc(sizeof(char*) * starchstrip_globals.chromosomes_num) );
+#else
+ starchstrip_globals.chromosomes_num = (size_t) delimiters_found + 1;
+ starchstrip_globals.chromosomes = malloc(sizeof(char*) * starchstrip_globals.chromosomes_num);
+#endif
+ for (size_t chr_idx = 0; chr_idx < starchstrip_globals.chromosomes_num; ++chr_idx) {
+ starchstrip_globals.chromosomes[chr_idx] = NULL;
+ }
+
+ /* populate chromosome name array */
+ size_t start = 0;
+ size_t end = 0;
+ int chr_idx = 0;
+ size_t chromosome_str_length = 0;
+ while (starchstrip_globals.chromosomes_str[end] != '\0') {
+ if (starchstrip_globals.chromosomes_str[end] == starchstrip_chromosomes_str_delimiter) {
+ chromosome_str_length = end - start;
+#ifdef __cplusplus
+ starchstrip_globals.chromosomes[chr_idx] = static_cast<char*>( malloc(chromosome_str_length + 1) );
+#else
+ starchstrip_globals.chromosomes[chr_idx] = (char*) malloc(chromosome_str_length + 1);
+#endif
+ memcpy(starchstrip_globals.chromosomes[chr_idx], starchstrip_globals.chromosomes_str + start, chromosome_str_length);
+ starchstrip_globals.chromosomes[chr_idx][chromosome_str_length] = '\0';
+ chr_idx++;
+ start = ++end;
+ }
+ end++;
+ }
+ chromosome_str_length = end - start;
+#ifdef __cplusplus
+ starchstrip_globals.chromosomes[chr_idx] = static_cast<char*>( malloc(chromosome_str_length + 1) );
+#else
+ starchstrip_globals.chromosomes[chr_idx] = (char*) malloc(chromosome_str_length + 1);
+#endif
+ memcpy(starchstrip_globals.chromosomes[chr_idx], starchstrip_globals.chromosomes_str + start, chromosome_str_length);
+ starchstrip_globals.chromosomes[chr_idx][chromosome_str_length] = '\0';
+
+ /* sort chromosome names */
+ qsort(starchstrip_globals.chromosomes, starchstrip_globals.chromosomes_num, sizeof(char*), STARCHSTRIP_compare_chromosome_names);
+
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_init_chromosomes_list() - exit ---\n");
+#endif
+}
+
+static int
+STARCHSTRIP_compare_chromosome_names(const void* a, const void* b)
+{
+#ifdef __cplusplus
+ return strcmp(*(const char**) a, *(const char**) b);
+#else
+ return strcmp(*(const char**) a, *(const char**) b);
+#endif
+}
+
+static void
+STARCHSTRIP_init_globals()
+{
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_init_globals() - enter ---\n");
+#endif
+
+ starchstrip_globals.chromosomes_str = NULL;
+ starchstrip_globals.chromosomes = NULL;
+ starchstrip_globals.chromosomes_num = 0;
+ starchstrip_globals.chromosomes_to_process = NULL;
+ starchstrip_globals.chromosomes_to_process_num = 0;
+ starchstrip_globals.inclusion_flag = kStarchFalse;
+ starchstrip_globals.exclusion_flag = kStarchFalse;
+ starchstrip_globals.cumulative_output_size = 0;
+ starchstrip_globals.output_records = NULL;
+ // --
+ starchstrip_globals.archive_fn = NULL;
+ starchstrip_globals.archive_metadata_json = NULL;
+ starchstrip_globals.archive_fp = NULL;
+ starchstrip_globals.archive_records = NULL;
+ starchstrip_globals.archive_type = STARCH_DEFAULT_COMPRESSION_TYPE;
+ starchstrip_globals.archive_version = NULL;
+ starchstrip_globals.archive_timestamp = NULL;
+ starchstrip_globals.archive_note = NULL;
+ starchstrip_globals.archive_metadata_offset = UINT64_C(0);
+ starchstrip_globals.archive_header_flag = kStarchFalse;
+ starchstrip_globals.archive_suppress_error_msgs = kStarchTrue;
+ starchstrip_globals.archive_preserve_json_ref = kStarchTrue;
+
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_init_globals() - exit ---\n");
+#endif
+}
+
+static void
+STARCHSTRIP_delete_globals()
+{
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_delete_globals() - enter ---\n");
+#endif
+
+ for (size_t idx = 0; idx < starchstrip_globals.chromosomes_num; ++idx) {
+ free(starchstrip_globals.chromosomes[idx]), starchstrip_globals.chromosomes[idx] = NULL;
+ }
+ free(starchstrip_globals.chromosomes), starchstrip_globals.chromosomes = NULL;
+ starchstrip_globals.chromosomes_num = 0;
+
+ free(starchstrip_globals.chromosomes_str), starchstrip_globals.chromosomes_str = NULL;
+
+ for (size_t idx = 0; idx < starchstrip_globals.chromosomes_to_process_num; ++idx) {
+ free(starchstrip_globals.chromosomes_to_process[idx]), starchstrip_globals.chromosomes_to_process[idx] = NULL;
+ }
+ free(starchstrip_globals.chromosomes_to_process), starchstrip_globals.chromosomes_to_process = NULL;
+ starchstrip_globals.chromosomes_to_process_num = 0;
+
+ free(starchstrip_globals.archive_fn), starchstrip_globals.archive_fn = NULL;
+ if (starchstrip_globals.archive_metadata_json) { json_decref(starchstrip_globals.archive_metadata_json), starchstrip_globals.archive_metadata_json = NULL; }
+ if (starchstrip_globals.archive_fp) { fclose(starchstrip_globals.archive_fp), starchstrip_globals.archive_fp = NULL; }
+ if (starchstrip_globals.archive_records) { STARCH_freeMetadata(&starchstrip_globals.archive_records), starchstrip_globals.archive_records = NULL; }
+ if (starchstrip_globals.archive_version) { free(starchstrip_globals.archive_version), starchstrip_globals.archive_version = NULL; }
+ if (starchstrip_globals.archive_timestamp) { free(starchstrip_globals.archive_timestamp), starchstrip_globals.archive_timestamp = NULL; }
+ if (starchstrip_globals.archive_note) { free(starchstrip_globals.archive_note), starchstrip_globals.archive_note = NULL; }
+
+ STARCH_freeMetadata(&starchstrip_globals.output_records);
+
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_delete_globals() - exit ---\n");
+#endif
+}
+
+static void
+STARCHSTRIP_init_command_line_options(int argc, char** argv)
+{
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_init_command_line_options() - enter ---\n");
+#endif
+
+ size_t optarg_length = 0;
+ int client_operation_count = 0;
+ int client_long_index;
+ int client_opt = getopt_long(argc,
+ argv,
+ starchstrip_opt_string,
+ starchstrip_long_options,
+ &client_long_index);
+
+ opterr = 0; /* disable error reporting by GNU getopt */
+
+ while (client_opt != -1) {
+ switch (client_opt) {
+ case 'i':
+ if (client_operation_count > 1) {
+ fprintf(stderr, "Error: Cannot specify both inclusion and exclusion operands\n");
+ STARCHSTRIP_print_usage(stderr);
+ exit(EINVAL); /* Invalid argument (POSIX.1) */
+ }
+ if (!optarg) {
+ fprintf(stderr, "Error: Chromosome list argument unspecified\n");
+ STARCHSTRIP_print_usage(stderr);
+ exit(EINVAL); /* Invalid argument (POSIX.1) */
+ }
+ optarg_length = strlen(optarg);
+#ifdef __cplusplus
+ starchstrip_globals.chromosomes_str = static_cast<char*>( malloc(optarg_length + 1) );
+#else
+ starchstrip_globals.chromosomes_str = malloc(optarg_length + 1);
+#endif
+ if (!starchstrip_globals.chromosomes_str) {
+ fprintf(stderr, "Error: Could not allocate space for chromosome list argument\n");
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ memcpy(starchstrip_globals.chromosomes_str, optarg, optarg_length + 1);
+ starchstrip_globals.inclusion_flag = kStarchTrue;
+ client_operation_count++;
+ break;
+ case 'x':
+ if (client_operation_count > 1) {
+ fprintf(stderr, "Error: Cannot specify both inclusion and exclusion operands\n");
+ STARCHSTRIP_print_usage(stderr);
+ exit(EINVAL); /* Invalid argument (POSIX.1) */
+ }
+ if (!optarg) {
+ fprintf(stderr, "Error: Chromosome list argument unspecified\n");
+ STARCHSTRIP_print_usage(stderr);
+ exit(EINVAL); /* Invalid argument (POSIX.1) */
+ }
+ optarg_length = strlen(optarg);
+#ifdef __cplusplus
+ starchstrip_globals.chromosomes_str = static_cast<char*>( malloc(optarg_length + 1) );
+#else
+ starchstrip_globals.chromosomes_str = malloc(optarg_length + 1);
+#endif
+ if (!starchstrip_globals.chromosomes_str) {
+ fprintf(stderr, "Error: Could not allocate space for chromosome list argument\n");
+ exit(ENOMEM); /* Not enough space (POSIX.1) */
+ }
+ memcpy(starchstrip_globals.chromosomes_str, optarg, optarg_length + 1);
+ starchstrip_globals.exclusion_flag = kStarchTrue;
+ client_operation_count++;
+ break;
+ case 'v':
+ STARCHSTRIP_print_version(stdout);
+ exit(EXIT_SUCCESS);
+ case 'h':
+ case '?':
+ STARCHSTRIP_print_usage(stdout);
+ exit(EXIT_SUCCESS);
+ default:
+ break;
+ }
+ client_opt = getopt_long(argc,
+ argv,
+ starchstrip_opt_string,
+ starchstrip_long_options,
+ &client_long_index);
+ }
+
+ if (argc != 4) {
+ fprintf(stderr, "Error: Starch file not specified\n");
+ STARCHSTRIP_print_usage(stderr);
+ exit(EINVAL); /* Invalid argument (POSIX.1) */
+ }
+ char* input_filename = argv[3];
+ if (!STARCHSTRIP_file_exists(input_filename)) {
+ fprintf(stderr, "Error: Starch file [%s] does not exist or is not accessible\n", input_filename);
+ STARCHSTRIP_print_usage(stderr);
+ exit(ENOENT); /* No such file or directory (POSIX.1) */
+ }
+ size_t input_filename_length = strlen(input_filename);
+#ifdef __cplusplus
+ starchstrip_globals.archive_fn = static_cast<char *>( malloc(input_filename_length + 1) );
+#else
+ starchstrip_globals.archive_fn = malloc(input_filename_length + 1);
+#endif
+ memcpy(starchstrip_globals.archive_fn, input_filename, input_filename_length + 1);
+
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_init_command_line_options() - exit ---\n");
+#endif
+}
+
+static void
+STARCHSTRIP_print_version(FILE* os)
+{
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_print_version() - enter ---\n");
+#endif
+
+ fprintf(os,
+ "%s\n" \
+ " citation: %s\n" \
+ " version: %s\n" \
+ " authors: %s\n",
+ name,
+ BEDOPS::citation(),
+ BEDOPS::revision(),
+ authors);
+
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_print_version() - exit ---\n");
+#endif
+}
+
+static void
+STARCHSTRIP_print_usage(FILE* os)
+{
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_print_usage() - enter ---\n");
+#endif
+
+ fprintf(os,
+ "%s\n" \
+ " citation: %s\n" \
+ " version: %s\n" \
+ " authors: %s\n" \
+ "%s\n",
+ name,
+ BEDOPS::citation(),
+ BEDOPS::revision(),
+ authors,
+ usage);
+
+#ifdef DEBUG
+ fprintf(stderr, "--- STARCHSTRIP_print_usage() - exit ---\n");
+#endif
+}
+
+static Boolean
+STARCHSTRIP_file_exists(char* fn)
+{
+ struct stat buf;
+ return stat(fn, &buf) == 0 ? kStarchTrue : kStarchFalse;
+}
+
+#ifdef __cplusplus
+} // namespace starch
+#endif
diff --git a/applications/bed/starch/src/starchstrip.h b/applications/bed/starch/src/starchstrip.h
new file mode 100644
index 0000000..6ed19a1
--- /dev/null
+++ b/applications/bed/starch/src/starchstrip.h
@@ -0,0 +1,143 @@
+//=========
+// Author: Alex Reynolds & Shane Neph
+// Project: starchstrip
+// File: starchstrip.h
+//=========
+
+//
+// BEDOPS
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+
+#ifndef STARCHSTRIP_H
+#define STARCHSTRIP_H
+
+#include <getopt.h>
+#include <inttypes.h>
+#include <errno.h>
+
+#include "data/starch/starchMetadataHelpers.h"
+
+#ifdef __cplusplus
+namespace {
+ using namespace starch;
+} // unnamed namespace
+#endif
+
+static const char *name = "starchstrip";
+static const char *authors = "Alex Reynolds and Shane Neph";
+static const char *usage = "\n" \
+ "USAGE: starchstrip [ --include | --exclude ] <chromosome-list> <starch-file>\n" \
+ " \n" \
+ " * Add either the --include or --exclude argument to filter the specified\n" \
+ " <starch-file> for chromosomes in <chromosome-list> for inclusion or\n" \
+ " exclusion, respectively. Note that you can only specify either inclusion\n" \
+ " or exclusion.\n\n" \
+ " * The <chromosome-list> argument is a comma-separated list of chromosome names\n" \
+ " to be included or excluded. This list is a *required* argument to either of the\n" \
+ " two --include and --exclude options.\n\n" \
+ " * The output is a Starch archive containing those chromosomes specified for inclusion\n" \
+ " or what chromosomes remain after exclusion from the original <starch-file>. A new\n" \
+ " metadata payload is appended to the output Starch archive.\n\n" \
+ " * The output is written to the standard output stream -- use the output redirection\n" \
+ " operator to write the result to a regular file, e.g.:\n\n" \
+ " $ starchstrip --exclude chrA,chrB,chrC in.starch > out.starch\n\n" \
+ " * If a specified chromosome is not in the input Starch archive, it will be ignored\n" \
+ " during processing.\n\n" \
+ " * Filtering simply copies over raw bytes from the input Starch archive and\n" \
+ " no extraction or recompression is performed. Use 'starchcat' to update the\n" \
+ " metadata, if new attributes are required.\n\n" \
+ " Process Flags\n" \
+ " --------------------------------------------------------------------------\n" \
+ " --include <chromosome-list> Include specified chromosomes from <starch-file>.\n\n" \
+ " --exclude <chromosome-list> Exclude specified chromosomes from <starch-file>.\n\n" \
+ " --version Show binary version.\n\n" \
+ " --help Show this usage message.\n";
+
+static struct globals {
+ char* chromosomes_str;
+ char** chromosomes;
+ size_t chromosomes_num;
+ char** chromosomes_to_process;
+ size_t chromosomes_to_process_num;
+ Boolean inclusion_flag;
+ Boolean exclusion_flag;
+ size_t cumulative_output_size;
+ Metadata* output_records;
+ // --
+ char* archive_fn;
+ json_t *archive_metadata_json;
+ FILE* archive_fp;
+ Metadata* archive_records;
+ CompressionType archive_type;
+ ArchiveVersion* archive_version;
+ char* archive_timestamp;
+ char* archive_note;
+ uint64_t archive_metadata_offset;
+ Boolean archive_header_flag;
+ Boolean archive_suppress_error_msgs;
+ Boolean archive_preserve_json_ref;
+} starchstrip_globals;
+
+static struct option starchstrip_long_options[] = {
+ { "include", required_argument, NULL, 'i' },
+ { "exclude", required_argument, NULL, 'x' },
+ { "version", no_argument, NULL, 'v' },
+ { "help", no_argument, NULL, 'h' },
+ { NULL, no_argument, NULL, 0 }
+};
+
+static const char *starchstrip_opt_string = "i:x:vh?";
+
+extern const char starchstrip_chromosomes_str_delimiter;
+extern const int starchstrip_archive_version_major_minimum;
+extern const int starchstrip_archive_version_minor_minimum;
+extern const int starchstrip_archive_version_revision_minimum;
+extern const size_t starchstrip_copy_buffer_size;
+
+const char starchstrip_chromosomes_str_delimiter = ',';
+const int starchstrip_archive_version_major_minimum = 2;
+const int starchstrip_archive_version_minor_minimum = 1;
+const int starchstrip_archive_version_revision_minimum = 0;
+const size_t starchstrip_copy_buffer_size = 65536;
+
+#ifdef __cplusplus
+namespace starch {
+#endif
+
+static void STARCHSTRIP_init_globals();
+static void STARCHSTRIP_delete_globals();
+static void STARCHSTRIP_init_command_line_options(int argc, char** argv);
+static void STARCHSTRIP_init_chromosomes_list();
+static int STARCHSTRIP_compare_chromosome_names(const void* a, const void* b);
+static void STARCHSTRIP_debug_chromosomes_to_query_list();
+static void STARCHSTRIP_debug_chromosomes_to_process_list();
+static void STARCHSTRIP_init_archive_metadata();
+static void STARCHSTRIP_check_archive_version();
+static void STARCHSTRIP_check_chromosome_stream_names();
+static void STARCHSTRIP_write_header(FILE* os);
+static void STARCHSTRIP_write_chromosome_streams(FILE* os);
+static void STARCHSTRIP_write_updated_metadata(FILE* os);
+static void STARCHSTRIP_print_version(FILE* os);
+static void STARCHSTRIP_print_usage(FILE* os);
+static Boolean STARCHSTRIP_file_exists(char* fn);
+
+#ifdef __cplusplus
+} // namespace starch
+#endif
+
+#endif
diff --git a/applications/bed/starch/src/unstarch.c b/applications/bed/starch/src/unstarch.c
index 5a77f95..1bf649e 100644
--- a/applications/bed/starch/src/unstarch.c
+++ b/applications/bed/starch/src/unstarch.c
@@ -6,7 +6,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -79,6 +79,7 @@ main(int argc, char **argv)
const Boolean preserveJSONRef = kStarchFalse; /* we generally do not want to preserve JSON reference */
char *jsonString = NULL;
unsigned char mdHashBuffer[STARCH2_MD_FOOTER_SHA1_LENGTH + 1] = {0};
+ Boolean signatureVerificationFlag = kStarchFalse;
/*
unstarch overview
@@ -189,6 +190,22 @@ main(int argc, char **argv)
resultValue = UNSTARCH_ELEMENT_NESTED_ALL_STR_ERROR;
break;
}
+ case UNSTARCH_SIGNATURE_ERROR: {
+ resultValue = UNSTARCH_SIGNATURE_ERROR;
+ break;
+ }
+ case UNSTARCH_SIGNATURE_VERIFY_ERROR: {
+ resultValue = UNSTARCH_SIGNATURE_VERIFY_ERROR;
+ break;
+ }
+ case UNSTARCH_ELEMENT_MAX_STRING_LENGTH_CHR_ERROR: {
+ resultValue = UNSTARCH_ELEMENT_MAX_STRING_LENGTH_CHR_ERROR;
+ break;
+ }
+ case UNSTARCH_ELEMENT_MAX_STRING_LENGTH_ALL_ERROR: {
+ resultValue = UNSTARCH_ELEMENT_MAX_STRING_LENGTH_ALL_ERROR;
+ break;
+ }
}
}
@@ -211,24 +228,26 @@ main(int argc, char **argv)
(resultValue == UNSTARCH_ELEMENT_NESTED_CHR_INT_ERROR) ||
(resultValue == UNSTARCH_ELEMENT_NESTED_ALL_INT_ERROR) ||
(resultValue == UNSTARCH_ELEMENT_NESTED_CHR_STR_ERROR) ||
- (resultValue == UNSTARCH_ELEMENT_NESTED_ALL_STR_ERROR) )
+ (resultValue == UNSTARCH_ELEMENT_NESTED_ALL_STR_ERROR) ||
+ (resultValue == UNSTARCH_ELEMENT_MAX_STRING_LENGTH_CHR_ERROR) ||
+ (resultValue == UNSTARCH_ELEMENT_MAX_STRING_LENGTH_ALL_ERROR))
{
if (STARCH_readJSONMetadata( &metadataJSON,
- &inFilePtr,
+ &inFilePtr,
#ifdef __cplusplus
- reinterpret_cast<const char *>( inFile ),
+ reinterpret_cast<const char *>( inFile ),
#else
- (const char *) inFile,
+ (const char *) inFile,
#endif
- &records,
- &type,
- &archiveVersion,
- &archiveTimestamp,
- ¬e,
- &metadataOffset,
- &headerFlag,
- suppressErrorMsgs,
- preserveJSONRef) != STARCH_EXIT_SUCCESS) {
+ &records,
+ &type,
+ &archiveVersion,
+ &archiveTimestamp,
+ ¬e,
+ &metadataOffset,
+ &headerFlag,
+ suppressErrorMsgs,
+ preserveJSONRef) != STARCH_EXIT_SUCCESS) {
fprintf(stderr, "ERROR: Could not read metadata\n");
resultValue = EXIT_FAILURE;
}
@@ -237,44 +256,47 @@ main(int argc, char **argv)
{
/* we suppress warnings from STARCH_readJSONMetadata() */
if (STARCH_readJSONMetadata( &metadataJSON,
- &inFilePtr,
+ &inFilePtr,
#ifdef __cplusplus
- reinterpret_cast<const char *>( inFile ),
+ reinterpret_cast<const char *>( inFile ),
#else
- (const char *) inFile,
+ (const char *) inFile,
#endif
- &records,
- &type,
- &archiveVersion,
- &archiveTimestamp,
- ¬e,
- &metadataOffset,
- &headerFlag,
- kStarchTrue,
- kStarchTrue) != STARCH_EXIT_SUCCESS)
+ &records,
+ &type,
+ &archiveVersion,
+ &archiveTimestamp,
+ ¬e,
+ &metadataOffset,
+ &headerFlag,
+ kStarchTrue,
+ kStarchTrue) != STARCH_EXIT_SUCCESS) {
fprintf(stdout, "0\n"); /* false -- no valid metadata, therefore not a starch archive */
- else
+ return EXIT_FAILURE;
+ }
+ else {
fprintf(stdout, "1\n"); /* true -- valid metadata, therefore a starch archive */
- return EXIT_SUCCESS;
+ return EXIT_SUCCESS;
+ }
}
- else if (resultValue == UNSTARCH_METADATA_SHA1_SIGNATURE_ERROR)
+ else if ( (resultValue == UNSTARCH_METADATA_SHA1_SIGNATURE_ERROR) || (resultValue == UNSTARCH_SIGNATURE_ERROR) || (resultValue == UNSTARCH_SIGNATURE_VERIFY_ERROR) )
{
if (STARCH_readJSONMetadata( &metadataJSON,
- &inFilePtr,
+ &inFilePtr,
#ifdef __cplusplus
- reinterpret_cast<const char *>( inFile ),
+ reinterpret_cast<const char *>( inFile ),
#else
- (const char *) inFile,
+ (const char *) inFile,
#endif
- &records,
- &type,
- &archiveVersion,
- &archiveTimestamp,
- ¬e,
- &metadataOffset,
- &headerFlag,
- suppressErrorMsgs,
- kStarchTrue) != STARCH_EXIT_SUCCESS) {
+ &records,
+ &type,
+ &archiveVersion,
+ &archiveTimestamp,
+ ¬e,
+ &metadataOffset,
+ &headerFlag,
+ suppressErrorMsgs,
+ kStarchTrue) != STARCH_EXIT_SUCCESS) {
fprintf(stderr, "ERROR: Could not read metadata\n");
resultValue = EXIT_FAILURE;
}
@@ -291,12 +313,12 @@ main(int argc, char **argv)
if (jsonString) {
#ifdef __cplusplus
STARCH_SHA1_All(reinterpret_cast<const unsigned char *>( reinterpret_cast<unsigned char *>( jsonString ) ),
- strlen(jsonString),
- mdHashBuffer);
+ strlen(jsonString),
+ mdHashBuffer);
#else
STARCH_SHA1_All((const unsigned char *) jsonString,
- strlen(jsonString),
- mdHashBuffer);
+ strlen(jsonString),
+ mdHashBuffer);
#endif
free(jsonString), jsonString = NULL;
json_decref(metadataJSON), metadataJSON = NULL;
@@ -722,10 +744,78 @@ main(int argc, char **argv)
}
break;
}
+ case UNSTARCH_SIGNATURE_ERROR: {
+ if ((archiveVersion->major == 2) && (archiveVersion->minor >= 2)) {
+ if (whichChromosome) {
+ UNSTARCH_printSignature(records, whichChromosome, mdHashBuffer);
+ }
+ else {
+ UNSTARCH_printMetadataSha1Signature(mdHashBuffer);
+ }
+ }
+ else {
+ fprintf(stderr, "ERROR: Archive version (%d.%d.%d) does not support per-chromosome signatures (starchcat or extract/recompress the archive to bring its version to v2.2.0 or greater)\n", archiveVersion->major, archiveVersion->minor, archiveVersion->revision);
+ resultValue = EXIT_FAILURE;
+ }
+ break;
+ }
+ case UNSTARCH_SIGNATURE_VERIFY_ERROR: {
+ if ((archiveVersion->major == 2) && (archiveVersion->minor >= 2)) {
+ if (whichChromosome) {
+ signatureVerificationFlag = UNSTARCH_verifySignature(&inFilePtr,
+ records,
+#ifdef __cplusplus
+ static_cast<const unsigned long long>( sizeof(starchRevision2HeaderBytes) ),
+#else
+ (const unsigned long long) sizeof(starchRevision2HeaderBytes),
+#endif
+ whichChromosome,
+ type);
+ }
+ else {
+ signatureVerificationFlag = UNSTARCH_verifyAllSignatures(&inFilePtr,
+ records,
+#ifdef __cplusplus
+ static_cast<const unsigned long long>( sizeof(starchRevision2HeaderBytes) ),
+#else
+ (const unsigned long long) sizeof(starchRevision2HeaderBytes),
+#endif
+ type);
+ }
+ }
+ else {
+ fprintf(stderr, "ERROR: Archive version (%d.%d.%d) does not support per-chromosome signature verification (starchcat or extract/recompress the archive to bring its version to v2.2.0 or greater)\n", archiveVersion->major, archiveVersion->minor, archiveVersion->revision);
+ resultValue = EXIT_FAILURE;
+ }
+ break;
+ }
+ case UNSTARCH_ELEMENT_MAX_STRING_LENGTH_CHR_ERROR: {
+ if ((archiveVersion->major == 2) && (archiveVersion->minor >= 2)) {
+ UNSTARCH_printLineMaxStringLengthForChromosome(records, whichChromosome);
+ }
+ else {
+ fprintf(stderr, "ERROR: Archive version (%d.%d.%d) does not support element maximum string length reporting (starchcat or extract/recompress the archive to bring its version to v2.2.0 or greater)\n", archiveVersion->major, archiveVersion->minor, archiveVersion->revision);
+ resultValue = EXIT_FAILURE;
+ }
+ break;
+ }
+ case UNSTARCH_ELEMENT_MAX_STRING_LENGTH_ALL_ERROR: {
+ if ((archiveVersion->major == 2) && (archiveVersion->minor >= 2)) {
+ UNSTARCH_printLineMaxStringLengthForAllChromosomes(records);
+ }
+ else {
+ fprintf(stderr, "ERROR: Archive version (%d.%d.%d) does not support element maximum string length reporting (starchcat or extract/recompress the archive to bring its version to v2.2.0 or greater)\n", archiveVersion->major, archiveVersion->minor, archiveVersion->revision);
+ resultValue = EXIT_FAILURE;
+ }
+ break;
+ }
}
}
- if ((resultValue == UNSTARCH_HELP_ERROR) ||
+ if ((resultValue == UNSTARCH_SIGNATURE_VERIFY_ERROR) && (signatureVerificationFlag == kStarchFalse)) {
+ resultValue = EXIT_FAILURE;
+ }
+ else if ((resultValue == UNSTARCH_HELP_ERROR) ||
(resultValue == UNSTARCH_VERSION_ERROR) ||
(resultValue == UNSTARCH_ARCHIVE_VERSION_ERROR) ||
(resultValue == UNSTARCH_ARCHIVE_CREATION_TIMESTAMP_ERROR) ||
@@ -745,8 +835,15 @@ main(int argc, char **argv)
(resultValue == UNSTARCH_ELEMENT_NESTED_CHR_INT_ERROR) ||
(resultValue == UNSTARCH_ELEMENT_NESTED_ALL_INT_ERROR) ||
(resultValue == UNSTARCH_ELEMENT_NESTED_CHR_STR_ERROR) ||
- (resultValue == UNSTARCH_ELEMENT_NESTED_ALL_STR_ERROR) )
+ (resultValue == UNSTARCH_ELEMENT_NESTED_ALL_STR_ERROR) ||
+ (resultValue == UNSTARCH_SIGNATURE_ERROR) ||
+ (resultValue == UNSTARCH_SIGNATURE_VERIFY_ERROR) ||
+ (resultValue == UNSTARCH_ELEMENT_MAX_STRING_LENGTH_CHR_ERROR) ||
+ (resultValue == UNSTARCH_ELEMENT_MAX_STRING_LENGTH_ALL_ERROR) )
+ {
resultValue = EXIT_SUCCESS;
+ }
+
/* cleanup */
if (option)
@@ -935,19 +1032,21 @@ UNSTARCH_parseCommandLineInputs(int argc, char **argv, char **chr, char **fn, ch
#endif
- if ((! *fn) ||
- (! *chr) ||
- (strcmp(*fn, "-") == 0) ||
- (strcmp(*fn, "--list") == 0) ||
- (strcmp(*fn, "--listJSON") == 0) ||
- (strcmp(*fn, "--list-json") == 0) ||
- (strcmp(*fn, "--list-json-no-trailing-newline") == 0) ||
- (strcmp(*fn, "--note") == 0) ||
- (strcmp(*fn, "--archive-type") == 0) ||
- (strcmp(*fn, "--archive-version") == 0) ||
- (strcmp(*fn, "--archive-timestamp") == 0) ||
- (strcmp(*fn, "--sha1-signature") == 0) ||
- (strcmp(*fn, "--is-starch") == 0) )
+ if ( (! *fn) ||
+ (! *chr) ||
+ (strcmp(*fn, "-") == 0) ||
+ (strcmp(*fn, "--list") == 0) ||
+ (strcmp(*fn, "--listJSON") == 0) ||
+ (strcmp(*fn, "--list-json") == 0) ||
+ (strcmp(*fn, "--list-json-no-trailing-newline") == 0) ||
+ (strcmp(*fn, "--note") == 0) ||
+ (strcmp(*fn, "--archive-type") == 0) ||
+ (strcmp(*fn, "--archive-version") == 0) ||
+ (strcmp(*fn, "--archive-timestamp") == 0) ||
+ (strcmp(*fn, "--sha1-signature") == 0) ||
+ (strcmp(*fn, "--signature") == 0) ||
+ (strcmp(*fn, "--verify-signature") == 0) ||
+ (strcmp(*fn, "--is-starch") == 0) )
{
if (ftr1)
free(ftr1);
@@ -988,6 +1087,14 @@ UNSTARCH_parseCommandLineInputs(int argc, char **argv, char **chr, char **fn, ch
*pval = UNSTARCH_METADATA_SHA1_SIGNATURE_ERROR;
return *pval;
}
+ else if (strcmp(*optn, "signature") == 0) {
+ *pval = UNSTARCH_SIGNATURE_ERROR;
+ return *pval;
+ }
+ else if (strcmp(*optn, "verify-signature") == 0) {
+ *pval = UNSTARCH_SIGNATURE_VERIFY_ERROR;
+ return *pval;
+ }
else if (strcmp(*optn, "note") == 0) {
*pval = UNSTARCH_ARCHIVE_NOTE_ERROR;
return *pval;
@@ -1004,6 +1111,10 @@ UNSTARCH_parseCommandLineInputs(int argc, char **argv, char **chr, char **fn, ch
*pval = (strcmp(*chr, "--elements") == 0) ? UNSTARCH_ELEMENT_COUNT_ALL_ERROR : UNSTARCH_ELEMENT_COUNT_CHR_ERROR;
return *pval;
}
+ else if (strcmp(*optn, "elements-max-string-length") == 0) {
+ *pval = (strcmp(*chr, "--elements-max-string-length") == 0) ? UNSTARCH_ELEMENT_MAX_STRING_LENGTH_ALL_ERROR : UNSTARCH_ELEMENT_MAX_STRING_LENGTH_CHR_ERROR;
+ return *pval;
+ }
else if (strcmp(*optn, "bases") == 0) {
*pval = (strcmp(*chr, "--bases") == 0) ? UNSTARCH_BASES_COUNT_ALL_ERROR : UNSTARCH_BASES_COUNT_CHR_ERROR;
return *pval;
@@ -1060,13 +1171,13 @@ UNSTARCH_printUsage(int errorType)
case UNSTARCH_HELP_ERROR:
default:
fprintf(stderr,
- "%s\n citation: %s\n binary version: %s (extracts archive version: %s or older)\n authors: %s\n%s\n",
- name,
- BEDOPS::citation(),
- BEDOPS::revision(),
- avStr,
- authors,
- usage);
+ "%s\n citation: %s\n binary version: %s (extracts archive version: %s or older)\n authors: %s\n%s\n",
+ name,
+ BEDOPS::citation(),
+ BEDOPS::revision(),
+ avStr,
+ authors,
+ usage);
break;
}
}
@@ -1090,10 +1201,10 @@ UNSTARCH_printRevision()
int result = sprintf(avStr, "%d.%d.%d", STARCH_MAJOR_VERSION, STARCH_MINOR_VERSION, STARCH_REVISION_VERSION);
if (result != -1)
fprintf(stdout,
- "%s\n binary version: %s (extracts archive version: %s or older)\n",
- name,
- BEDOPS::revision(),
- avStr);
+ "%s\n binary version: %s (extracts archive version: %s or older)\n",
+ name,
+ BEDOPS::revision(),
+ avStr);
free(avStr);
}
}
@@ -1106,11 +1217,11 @@ UNSTARCH_printArchiveVersion(const ArchiveVersion *av)
#endif
if (av)
fprintf(stdout,
- "%s\n archive version: %d.%d.%d\n",
- name,
- av->major,
- av->minor,
- av->revision);
+ "%s\n archive version: %d.%d.%d\n",
+ name,
+ av->major,
+ av->minor,
+ av->revision);
}
void
@@ -1121,8 +1232,8 @@ UNSTARCH_printArchiveTimestamp(const char *at)
#endif
if (at)
fprintf(stdout,
- "%s\n",
- at);
+ "%s\n",
+ at);
}
void
@@ -1133,8 +1244,8 @@ UNSTARCH_printNote(const char *note)
#endif
if (note)
fprintf(stdout,
- "%s\n",
- note);
+ "%s\n",
+ note);
}
void
@@ -1172,18 +1283,18 @@ UNSTARCH_printMetadataSha1Signature(unsigned char *sha1Buffer)
#ifdef __cplusplus
STARCH_encodeBase64(&jsonBase64String,
- static_cast<const size_t>( STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH ),
- const_cast<const unsigned char *>( sha1Buffer ),
- static_cast<const size_t>( sha1BufferLength ));
+ static_cast<const size_t>( STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH ),
+ const_cast<const unsigned char *>( sha1Buffer ),
+ static_cast<const size_t>( sha1BufferLength ));
#else
STARCH_encodeBase64(&jsonBase64String,
- (const size_t) STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH,
- (const unsigned char *) sha1Buffer,
- (const size_t) sha1BufferLength);
+ (const size_t) STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH,
+ (const unsigned char *) sha1Buffer,
+ (const size_t) sha1BufferLength);
#endif
if (!jsonBase64String) {
- fprintf(stderr, "ERROR: Could not allocate space for base64-encoded metadata string representation\n");
+ fprintf(stderr, "ERROR: Could not allocate space for Base64-encoded metadata string representation\n");
exit(-1);
}
fprintf(stdout, "%s\n", jsonBase64String);
diff --git a/applications/bed/starch/src/unstarch.h b/applications/bed/starch/src/unstarch.h
index f8a7d92..24fac49 100644
--- a/applications/bed/starch/src/unstarch.h
+++ b/applications/bed/starch/src/unstarch.h
@@ -7,7 +7,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -32,40 +32,91 @@
static const char *name = "unstarch";
static const char *authors = "Alex Reynolds and Shane Neph";
static const char *usage = "\n" \
- "USAGE: unstarch [ <chromosome> ] [ --elements | --bases | --bases-uniq | --has-duplicates | --has-nested | --list | --list-json | --list-chromosomes | --archive-timestamp | --note | --archive-version | --is-starch ] <starch-file>\n" \
+ "USAGE: unstarch [ <chromosome> ] [ --elements | \n" \
+ " --elements-max-string-length |\n" \
+ " --bases | --bases-uniq |\n" \
+ " --has-duplicates | --has-nested | --list |\n" \
+ " --list-json | --list-chromosomes |\n" \
+ " --archive-timestamp | --note |\n" \
+ " --archive-version | --is-starch |\n" \
+ " --signature | --verify-signature ]\n" \
+ " <starch-file>\n" \
"\n" \
- " Process Flags:\n\n" \
- " <chromosome> Optional. Either unarchives chromosome-specific records from the starch archive file or restricts action of operator to chromosome (e.g., chr1, chrY, etc.).\n" \
- " --elements Show total element count for archive. If <chromosome> is specified, the result shows the element count for the chromosome.\n" \
+ " Modifiers\n" \
+ " --------------------------------------------------------------------------\n" \
+ " <chromosome> Optional. Either unarchives chromosome-\n" \
+ " specific records from the starch archive\n" \
+ " file or restricts action of operator to\n" \
+ " chromosome (e.g., chr1, chrY, etc.).\n\n" \
+ " Process Flags\n" \
+ " --------------------------------------------------------------------------\n" \
+ " --elements Show total element count for archive. If\n" \
+ " <chromosome> is specified, the result\n" \
+ " shows the element count for the\n" \
+ " chromosome.\n\n" \
+ " --elements-max-string-length Show the maximum string length over all\n" \
+ " elements in <chromosome>, if specified.\n" \
+ " If <chromosome> is not specified, the\n" \
+ " maximum string length is shown over all\n" \
+ " chromosomes.\n\n" \
" --bases,\n" \
- " --bases-uniq Show total and unique base counts, respectively, for archive. If <chromosome> is specified, the count is specific to the chromosome, if available.\n" \
- " --has-duplicate, Show whether there is one or more duplicate elements in the specified chromosome, either as a numerical (1/0) or string (true/false) value. If no <chromosome> is specified, the value given indicates if there is one or more duplicate elements across all chromosome records.\n" \
- " --has-duplicate-as-string \n" \
- " --has-nested, Show whether there is one ore more nested elements in the specified chromosome, either as a numerical (1/0) or string (true/false) value. If no <chromosome> is specified, the value given indicates if there is one or more nested elements across all chromosome records.\n" \
- " --has-nested-as-string \n" \
- " --list List archive metadata (output is in text format). If chromosome is specified, the attributes of the given chromosome are shown.\n" \
- " --list-json, List archive metadata (output is in JSON format)\n" \
- " --list-json-no-trailing-newline \n" \
- " --list-chr, List all or specified chromosome in starch archive (similar to \"bedextract --list-chr\"). If <chromosome> is specified but is not in the output list, nothing is returned.\n" \
- " --list-chromosomes \n" \
- " --note Show descriptive note, if available.\n" \
- " --sha1-signature Show SHA1 signature of JSON-formatted metadata (Base64-encoded).\n" \
- " --archive-timestamp Show archive creation timestamp (ISO 8601 format).\n" \
- " --archive-type Show archive compression type.\n" \
- " --archive-version Show archive version.\n" \
- " --is-starch Test if <starch-file> is a valid archive and print 0/1 (false/true) to standard output.\n" \
- " --version Show binary version.\n" \
+ " --bases-uniq Show total and unique base counts,\n" \
+ " respectively, for archive. If\n" \
+ " <chromosome> is specified, the count is\n"
+ " specific to the chromosome, if available.\n\n" \
+ " --has-duplicate-as-string, \n" \
+ " --has-duplicate Show whether there is one or more\n" \
+ " duplicate elements in the specified\n" \
+ " chromosome, either as a numerical (1/0)\n" \
+ " or string (true/false) value. If no\n" \
+ " <chromosome> is specified, the value\n" \
+ " given indicates if there is one or more\n" \
+ " duplicate elements across all chromosome\n" \
+ " records.\n\n" \
+ " --has-nested-as-string, \n" \
+ " --has-nested Show whether there is one ore more nested\n" \
+ " elements in the specified chromosome,\n" \
+ " either as a numerical (1/0) or string\n" \
+ " (true/false) value. If no <chromosome> is\n" \
+ " specified, the value given indicates if\n" \
+ " there is one or more nested elements\n" \
+ " across all chromosome records.\n\n" \
+ " --list List archive metadata (output is in text\n" \
+ " format). If chromosome is specified, the\n" \
+ " attributes of the given chromosome are\n" \
+ " shown.\n\n" \
+ " --list-json, \n" \
+ " --list-json-no-trailing-newline List archive metadata (output is in JSON\n" \
+ " format)\n\n" \
+ " --list-chr, \n" \
+ " --list-chromosomes List all or specified chromosome in\n" \
+ " starch archive (like \"bedextract --list-\n" \
+ " chr\"). If <chromosome> is specified but\n" \
+ " is not in the output list, nothing is\n" \
+ " returned.\n\n" \
+
+ " --note Show descriptive note, if available.\n\n" \
+ " --signature Display the Base64-encoded SHA-1 data\n" \
+ " integrity signature for specified\n" \
+ " <chromosome>, or the signatures of the\n" \
+ " metadata and all available chromosomes,\n" \
+ " if the <chromosome> is unspecified.\n\n" \
+ " --verify-signature Verify data integrity of specified\n" \
+ " <chromosome>, or the integrity of all\n" \
+ " available chromosomes, if the\n" \
+ " <chromosome> is unspecified.\n\n" \
+ " --archive-timestamp Show archive creation timestamp (ISO 8601\n" \
+ " format).\n\n" \
+ " --archive-type Show archive compression type.\n\n" \
+ " --archive-version Show archive version.\n\n" \
+ " --is-starch Test if <starch-file> is a valid archive\n" \
+ " and print 0/1 (false/true) to standard\n" \
+ " output. Unstarch will also return a non-\n" \
+ " zero error code if the input file is not\n" \
+ " a valid archive.\n\n" \
+ " --version Show binary version.\n\n" \
" --help Show this usage message.\n";
-/*
- On Darwin, file I/O is 64-bit by default (OS X 10.5 at least) so we use standard
- types and calls
-*/
-
-#ifdef __APPLE__
-#define off64_t off_t
-#endif
-
#ifdef __cplusplus
namespace starch {
#endif
diff --git a/docs/_static/custom.css b/docs/_static/custom.css
index 3d60d63..12a52ee 100644
--- a/docs/_static/custom.css
+++ b/docs/_static/custom.css
@@ -23,7 +23,7 @@ h2 {
}
tt {
- font-size: 0.9em;
+ font-size: 1em;
}
.index_cols {
@@ -36,7 +36,7 @@ tt {
}
.col_element {
- display: inline-block;
+ display: block;
}
@media only screen and (min-device-width : 320px) and (max-device-width : 568px) and (orientation : landscape) {
@@ -87,3 +87,11 @@ ul {
background-color: #d0e8ff;
border: 1px solid #97bee6;
}
+
+ul.simple > li {
+ margin-bottom: 10px;
+}
+
+.section > ul > li {
+ margin-bottom: 10px;
+}
\ No newline at end of file
diff --git a/docs/conf.py b/docs/conf.py
index 8da62f4..5273a2a 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -26,8 +26,8 @@ sys.path.insert(0, os.path.abspath('.'))
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode']
-#, 'sphinx.ext.mathbase']
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode']
+#, 'sphinx.ext.mathbase', 'sphinx.ext.pngmath', 'sphinx.ext.imgmath']
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
@@ -43,7 +43,7 @@ master_doc = 'index'
# General information about the project.
project = u'BEDOPS'
-copyright = u'2011-2016, Shane Neph, Alex Reynolds'
+copyright = u'2011-2017, Shane Neph, Alex Reynolds'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@@ -52,7 +52,7 @@ copyright = u'2011-2016, Shane Neph, Alex Reynolds'
# The short X.Y version.
version = '2.4'
# The full version, including alpha/beta/rc tags.
-release = '2.4.20'
+release = '2.4.26'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
diff --git a/docs/content/reference/file-management/compression.rst b/docs/content/reference/file-management/compression.rst
index 42d30e9..1519e13 100644
--- a/docs/content/reference/file-management/compression.rst
+++ b/docs/content/reference/file-management/compression.rst
@@ -6,4 +6,7 @@ Compression
compression/starch
compression/unstarch
compression/starchcat
+ compression/starchsplit
compression/starch-specification
+ compression/starch-diff
+
\ No newline at end of file
diff --git a/docs/content/reference/file-management/compression/starch-diff.rst b/docs/content/reference/file-management/compression/starch-diff.rst
new file mode 100644
index 0000000..03bba1f
--- /dev/null
+++ b/docs/content/reference/file-management/compression/starch-diff.rst
@@ -0,0 +1,52 @@
+.. _starch_diff:
+
+`starch-diff`
+=============
+
+This tool allows the end user to quickly compare the compressed chromosomes in two or more v2.2+ Starch archives.
+
+==================
+Inputs and outputs
+==================
+
+-----
+Input
+-----
+
+The `starch-diff` utility takes in two or more Starch v2.2+ archives as input. The end user may also add `--chr <chr>` to compare one chromosome directly; otherwise, all chromosomes in specified archives are compared.
+
+------
+Output
+------
+
+The typical output of `starch-diff` is a message indicating the archives' chromosome(s) are identical or dissimilar.
+
+In addition, if the chromosomes are identical, `starch-diff` exits with a zero status code. Likewise, if any chromosomes are dissimilar, `starch-diff` exits with a non-zero status code.
+
+============
+Requirements
+============
+
+If the user passes in a pre-v2.2 archive, the utility will exit with a fatal error.
+
+=====
+Usage
+=====
+
+Use the ``--help`` option to list all options:
+
+::
+
+ starch-diff
+ citation: http://bioinformatics.oxfordjournals.org/content/28/14/1919.abstract
+ version: 2.4.26
+ authors: Alex Reynolds and Shane Neph
+
+ $ starch-diff [ --chr <chr> ] starch-file-1 starch-file-2 [ starch-file-3 ... ]
+
+ The 'starch-diff' utility compares the signatures of two or more specified
+ Starch v2.2+ archives for all chromosomes, or for a specified chromosome.
+
+
+.. role:: bash(code)
+ :language: bash
diff --git a/docs/content/reference/file-management/compression/starch-specification.rst b/docs/content/reference/file-management/compression/starch-specification.rst
index 7f3e202..6128585 100644
--- a/docs/content/reference/file-management/compression/starch-specification.rst
+++ b/docs/content/reference/file-management/compression/starch-specification.rst
@@ -79,7 +79,7 @@ The archive key scheme is described below:
"type": "starch",
"customUCSCHeaders": (Boolean),
"creationTimestamp": (string),
- "version": { "major": 2, "minor": 1, "revision": 0 },
+ "version": { "major": 2, "minor": 2, "revision": 0 },
"compressionFormat": (unsigned integer),
"note": (string, optional)
},
@@ -92,7 +92,9 @@ The ``customUCSCHeaders`` value is either ``true`` or ``false``. If ``true``, th
The ``creationTimestamp`` value is an `ISO 8601 <http://en.wikipedia.org/wiki/ISO-8601>`_ string that specifies the creation date and time of the archive. Most scripting and programming languages can parse ISO 8601-formatted date strings with little or no extra work.
-The ``version`` is a triplet of integer values specifying the version of the archive. For a v2.x archive, the major version will be set to ``2``. Major, minor and revision values need not necessarily be the identical to the version of the :ref:`starch` binary used to create the archive. At this time (April 2014), we offer v2 and v2.1 archives: each make different stream metadata fields available.
+The ``version`` is a triplet of integer values specifying the version of the archive. For a v2.x archive, the major version will be set to ``2``. Major, minor and revision values need not necessarily be the identical to the version of the :ref:`starch` binary used to create the archive.
+
+At this time (November 2016), we offer v2.0, v2.1, and v2.2 archives: Each version makes different stream metadata fields available.
The ``compressionFormat`` key specifies the backend compression format used for the chromosome streams contained within the archive. We currently use ``0`` to specify ``bzip2`` and ``1`` to specify ``gzip``. No other backend formats are available at this time.
@@ -119,7 +121,9 @@ The ``streams`` key scheme contains an array of objects, each describing the att
"nonUniqueBaseCount": (unsigned integer),
"uniqueBaseCount": (unsigned integer),
"duplicateElementExists": (Boolean),
- "nestedElementExists": (Boolean)
+ "nestedElementExists": (Boolean),
+ "signature": (string),
+ "uncompressedLineMaxStringLength": (integer)
},
...
]
@@ -141,6 +145,10 @@ The ``duplicateElementExists`` key specifies if there is a duplicate BED element
The ``nestedElementExists`` key specifies if there is a nested BED element somewhere within the compressed chromosome stream. Refer to BEDOPS documentation to see how :ref:`nested elements <nested_elements>` are defined.
+The ``signature`` key, available in v2.2 archives, specifies the Base64-encoded SHA-1 data integrity signature generated from the transformed chromosome stream (not the raw BED data, but the reduced or transformed form that is compressed). This can be used to compare the transformed bytes for chromosomes from different archives, or to validate the genomic data in a Starch archive by chromosome, or in entirety. (Note that, if ``--omit-signature`` was used to create a v2.2 archive, this ke [...]
+
+The ``uncompressedLineMaxStringLength`` key, available in v2.2 archives, specifies the maximum string length over all records in the chromosome stream.
+
.. _starch_archive_metadata_offset:
------
@@ -163,7 +171,7 @@ Hash
The metadata hash is a 28-byte long, `Base64 <http://en.wikipedia.org/wiki/Base64>`_ -encoded `SHA-1 <http://en.wikipedia.org/wiki/SHA-1#Data_Integrity>`_ hash of the bytes that make up the JSON-formatted metadata string.
-This data is used to validate the integrity of the metadata: Any change to the metadata (*e.g.*, data corruption that changes stream offset values) causes :ref:`unstarch` and other Starch utilities and applications to exit early with a fatal, informative error.
+This data is used to validate the integrity of the metadata: Any change to the metadata (*e.g.*, data corruption that changes stream offset values, or the data integrity signatures for each chromosome stream) causes :ref:`unstarch` and other Starch utilities and applications to exit early with a fatal, informative error.
.. _starch_archive_padding:
diff --git a/docs/content/reference/file-management/compression/starch.rst b/docs/content/reference/file-management/compression/starch.rst
index 8f3c4b4..6e1a6ab 100644
--- a/docs/content/reference/file-management/compression/starch.rst
+++ b/docs/content/reference/file-management/compression/starch.rst
@@ -35,7 +35,7 @@ This utility outputs a :ref:`Starch v2-formatted <starch_specification>` archive
Requirements
============
-The :ref:`starch` tool requires data in a relaxed variation of the BED format as described by `UCSC’s browser documentation <http://genome.ucsc.edu/FAQ/FAQformat.html#format1>`_. BED data should be sorted before compression, *e.g.* with BEDOPS :ref:`sort-bed`.
+The :ref:`starch` tool requires data in a relaxed variation of the BED format as described by `UCSC’s browser documentation <http://genome.ucsc.edu/FAQ/FAQformat.html#format1>`_. BED data should be sorted before compression, *i.e.* with BEDOPS :ref:`sort-bed`.
At a minimum, three columns are required to specify the chromosome name and start and stop positions. Additional columns may be specified, containing up to 128 kB of data per row (including tab delimiters).
@@ -49,24 +49,43 @@ Use the ``--help`` option to list all options:
starch
citation: http://bioinformatics.oxfordjournals.org/content/28/14/1919.abstract
- binary version: 2.4.20 (creates archive version: 2.1.0)
+ binary version: 2.4.26 (creates archive version: 2.2.0)
authors: Alex Reynolds and Shane Neph
- USAGE: starch [--note="foo bar..."] [--bzip2 | --gzip] [--header] [<unique-tag>] <bed-file>
-
+ USAGE: starch [ --note="foo bar..." ]
+ [ --bzip2 | --gzip ]
+ [ --omit-signature ]
+ [ --report-progress=N ]
+ [ --header ] [ <unique-tag> ] <bed-file>
+
* BED input must be sorted lexicographically (e.g., using BEDOPS sort-bed).
* Please use '-' to indicate reading BED data from standard input.
* Output must be directed to a regular file.
- * The bzip2 compression type makes smaller archives, while gzip extracts faster.
-
- Process Flags:
+ * The bzip2 compression type makes smaller archives, while gzip extracts
+ faster.
+
+ Process Flags
+ --------------------------------------------------------------------------
+ --note="foo bar..." Append note to output archive metadata (optional).
- --note="foo bar..." Append note to output archive metadata (optional)
- --bzip2 | --gzip Specify backend compression type (optional, default is bzip2)
- --header Support BED input with custom UCSC track, SAM or VCF headers, or generic comments (optional)
- <unique-tag> Specify unique identifier for transformed data (optional)
- --help Show this usage message
- --version Show binary version
+ --bzip2 | --gzip Specify backend compression type (optional, default
+ is bzip2).
+
+ --omit-signature Skip generating per-chromosome data integrity signature
+ (optional, default is to generate signature).
+
+ --report-progress=N Report compression progress every N elements per
+ chromosome to standard error stream (optional)
+
+ --header Support BED input with custom UCSC track, SAM or VCF
+ headers, or generic comments (optional).
+
+ <unique-tag> Optional. Specify unique identifier for transformed
+ data.
+
+ --version Show binary version.
+
+ --help Show this usage message.
=======
Options
@@ -88,6 +107,22 @@ Use the ``--note="xyz..."`` option to add a custom string that describes the arc
.. note:: The only limitation on the length of a note is the command-line shell's maximum argument length parameter (as found on most UNIX systems with the command ``getconf ARG_MAX``) minus the length of the non- ``--note="..."`` command components. On most desktop systems, this value will be approximately 256 kB.
+---------------------------------------
+Per-chromosome data integrity signature
+---------------------------------------
+
+By default, a data integrity signature is generated for each chromosome. This can be used to verify if chromosome streams from two or more Starch archives are identical, or used to test the integrity of a chromosome, to identify potential data corruption.
+
+Generating this signature adds to the computational cost of compression, or an integrity signature may not be useful for all archives. Add the ``--omit-signature`` option, if the compression time is too high or the data integrity signature is not needed.
+
+--------------------
+Compression progress
+--------------------
+
+To optionally track the progress of compression, use the ``--report-progress=N`` option, specifying a positive integer ``N`` to report the compression of the *N* -th element for the current chromosome. The report is printed to the standard error stream.
+
+.. note:: For instance, specifying a value of ``1`` reports the compression of every input element of all chromosomes, while a value of ``1000`` would report the compression of every 1000th element of the current chromosome.
+
-------
Headers
-------
diff --git a/docs/content/reference/file-management/compression/starchcat.rst b/docs/content/reference/file-management/compression/starchcat.rst
index fc258e8..bdfcb29 100644
--- a/docs/content/reference/file-management/compression/starchcat.rst
+++ b/docs/content/reference/file-management/compression/starchcat.rst
@@ -56,26 +56,49 @@ Use the ``--help`` option to list all options:
::
starchcat
- citation: http://bioinformatics.oxfordjournals.org/content/28/14/1919.abstract
- version: 2.4.20
- authors: Alex Reynolds and Shane Neph
+ citation: http://bioinformatics.oxfordjournals.org/content/28/14/1919.abstract
+ version: 2.4.26
+ authors: Alex Reynolds and Shane Neph
- USAGE: starchcat [ --note="..." ] [ --bzip2 | --gzip ] <starch-file-1> [<starch-file-2> ...]
-
- * At least one lexicographically-sorted, headerless starch archive is required.
- While two or more inputs make sense for a multiset union operation, you can starchcat
- one file in order to update its metadata, recompress it with a different backend method,
- or add a note annotation.
+ USAGE: starchcat [ --note="..." ]
+ [ --bzip2 | --gzip ]
+ [ --omit-signature ]
+ [ --report-progress=N ] <starch-file-1> [<starch-file-2> ...]
- * Compressed data are sent to standard output. Use the '>' operator to redirect
- to a file.
+ * At least one lexicographically-sorted, headerless starch archive is
+ required.
- Process Flags:
+ * While two or more inputs make sense for a multiset union operation, you
+ can starchcat one file in order to update its metadata, recompress it
+ with a different backend method, or add a note annotation.
- --note="foo bar..." Append note to output archive metadata (optional)
- --bzip2 | --gzip Specify backend compression type (optional, default is bzip2)
- --version Show binary version
- --help Show this usage message
+ * Compressed data are sent to standard output. Use the '>' operator to
+ redirect to a file.
+
+ Process Flags
+ --------------------------------------------------------------------------
+ --note="foo bar..." Append note to output archive metadata (optional).
+
+ --bzip2 | --gzip Specify backend compression type (optional, default
+ is bzip2).
+
+ --omit-signature Skip generating per-chromosome data integrity signature
+ (optional, default is to generate signature).
+
+ --report-progress=N Report compression progress every N elements per
+ chromosome to standard error stream (optional)
+
+ --version Show binary version.
+
+ --help Show this usage message.
+
+---------------------------------------
+Per-chromosome data integrity signature
+---------------------------------------
+
+By default, a data integrity signature is generated for each chromosome. This can be used to verify if chromosome streams from two or more Starch archives are identical, or used to test the integrity of a chromosome, to identify potential data corruption.
+
+Generating this signature adds to the computational cost of compression, or an integrity signature may not be useful for all archives. Add the ``--omit-signature`` option, if the compression time is too high or the data integrity signature is not needed.
-------
Example
@@ -89,7 +112,7 @@ Let's say we have a set of 23 :ref:`starch` archives, one for each chromosome of
The :ref:`starchcat` utility parses the metadata from each of the 23 inputs, determines what data to either simple copy or reprocess, and then it performs the merge. Cleanup is performed afterwards, as necessary, and the output is a brand new :ref:`starch` file, written to ``humanGenome.starch``.
-.. note:: No filtering or processing is performed on extracted BED elements, before they are written to the final output. Thus, *it is possible for duplicate BED elements to occur*.
+.. note:: No filtering or processing is performed on extracted BED elements, before they are written to the final output. Thus, *it is possible for duplicate BED elements to occur*. It would be easy to use the ``--signature`` option to validate the expected content of a new Starch archive.
However, the final archive is sorted per :ref:`sort-bed` ordering, so that data extracted from this archive will be ready for use with BEDOPS utilities.
diff --git a/docs/content/reference/file-management/compression/starchstrip.rst b/docs/content/reference/file-management/compression/starchstrip.rst
new file mode 100644
index 0000000..5113f98
--- /dev/null
+++ b/docs/content/reference/file-management/compression/starchstrip.rst
@@ -0,0 +1,101 @@
+.. _starchstrip:
+
+`starchstrip`
+=============
+
+The ``starchstrip`` utility efficiently pulls out per-chromosome records contained within a BEDOPS :ref:`Starch-formatted <starch_specification>` archive and writes the filtered result to a new Starch archive. This utility allows either exclusion or inclusion of one or more specified chromosome names.
+
+Previously, it would be necessary to extract records with :ref:`unstarch <unstarch>`, use `awk` or similar to filter down to the desired set of records, and recompress with :ref:`starch <starch>`. In contrast, ``starchstrip`` identifies just the pieces of data of interest within an archive and writes them to a new archive, with an updated metadata payload, avoiding the need for costly and wasteful extraction and re-compression.
+
+==================
+Inputs and outputs
+==================
+
+-----
+Input
+-----
+
+The input to :ref:`starchstrip` consists of a BEDOPS :ref:`Starch-formatted <starch_specification>` archive file, along with the specification of either ``--include`` or ``--exclude`` for inclusion or exlusion of chromosome records from the archive. One or more chromosome names are provided as a comma-separated string.
+
+.. note:: If the chromosome listing contains chromosome names not in the input archive, they will be ignored.
+
+------
+Output
+------
+
+The :ref:`starchstrip` tool writes a :ref:`starch` -formatted archive to the standard output stream, which is usually redirected to a regular file. The output contains the same compressed data from the original file (no extraction or recompression is performed) and so preserves the archive version, compression type, and other archive attributes.
+
+.. note:: If the archive's metadata attributes need updating (to gain updated metadata features, for instance, such as data integrity signatures), the :ref:`starchcat <starchcat>` utility should be used to update older archives.
+
+.. note:: If the specified combination of operation and chromosome names would result in output that is identical to the original file, or output that would be an empty file, ``starchstrip`` will exit early with a fatal error.
+
+=====
+Usage
+=====
+
+Use the ``--help`` option to list all options:
+
+::
+
+ starchstrip
+ citation: http://bioinformatics.oxfordjournals.org/content/28/14/1919.abstract
+ version: 2.4.26
+ authors: Alex Reynolds and Shane Neph
+
+ USAGE: starchstrip [ --include | --exclude ] <chromosome-list> <starch-file>
+
+ * Add either the --include or --exclude argument to filter the specified
+ <starch-file> for chromosomes in <chromosome-list> for inclusion or
+ exclusion, respectively. Note that you can only specify either inclusion
+ or exclusion.
+
+ * The <chromosome-list> argument is a comma-separated list of chromosome names
+ to be included or excluded. This list is a *required* argument to either of the
+ two --include and --exclude options.
+
+ * The output is a Starch archive containing those chromosomes specified for inclusion
+ or what chromosomes remain after exclusion from the original <starch-file>. A new
+ metadata payload is appended to the output Starch archive.
+
+ * The output is written to the standard output stream -- use the output redirection
+ operator to write the result to a regular file, e.g.:
+
+ $ starchstrip --exclude chrN in.starch > out.starch
+
+ * Filtering simply copies over raw bytes from the input Starch archive and
+ no extraction or recompression is performed. Use 'starchcat' to update the
+ metadata, if new attributes are required.
+
+ Process Flags
+ --------------------------------------------------------------------------
+ --include <chromosome-list> Include specified chromosomes from <starch-file>.
+
+ --exclude <chromosome-list> Exclude specified chromosomes from <starch-file>.
+
+ --version Show binary version.
+
+ --help Show this usage message.
+
+-------
+Example
+-------
+
+Let's say we have an archive containing 23 chromosomes, one for each of the human genome: ``chr1``, ``chr2``, and so on, to ``chrY``. (To simplify this example, we leave out mitochondrial, random, pseudo- and other chromosomes.) As an example, say we want a new Starch archive that contains chromosomes ``chr4``, ``chr8``, and ``chr17``. We can use ``starchstrip`` to efficiently write out a new archive with just those three chromosomes:
+
+::
+
+ $ starchstrip --include chr4,chr8,chr17 humanGenome.starch > humanGenome.chrs4_8_and_17.starch
+
+The :ref:`starchstrip` utility parses the metadata from the input ``humanGenome.starch`` and uses its details to decide how to write out the subset of chromosomes, along with a metadata payload specific to the three chromosomes. No extraction or recompression is performed; this is as fast as copying just the parts of the file we are interested in.
+
+As a second example, we can instead use the ``--exclude`` operand to copy over all chromosomes *except* those we choose. To continue the example above, we can get the "inverse" of ``humanGenome.chrs4_8_and_17.starch`` with the following:
+
+::
+
+ $ starchstrip --exclude chr4,chr8,chr17 humanGenome.starch > humanGenome.all_chrs_except_chrs4_8_and_17.starch
+
+
+
+.. |--| unicode:: U+2013 .. en dash
+.. |---| unicode:: U+2014 .. em dash, trimming surrounding whitespace
+ :trim:
\ No newline at end of file
diff --git a/docs/content/reference/file-management/compression/unstarch.rst b/docs/content/reference/file-management/compression/unstarch.rst
index cb6fabc..294f359 100644
--- a/docs/content/reference/file-management/compression/unstarch.rst
+++ b/docs/content/reference/file-management/compression/unstarch.rst
@@ -43,33 +43,108 @@ Use the ``--help`` option to list all options:
unstarch
citation: http://bioinformatics.oxfordjournals.org/content/28/14/1919.abstract
- binary version: 2.4.20 (extracts archive version: 2.1.0 or older)
+ binary version: 2.4.26 (extracts archive version: 2.2.0 or older)
authors: Alex Reynolds and Shane Neph
- USAGE: unstarch [ <chromosome> ] [ --elements | --bases | --bases-uniq | --duplicatesExist | --nestedsExist | --list | --list-json | --list-chromosomes | --archive-timestamp | --note | --archive-version | --is-starch ] <starch-file>
+ USAGE: unstarch [ <chromosome> ] [ --elements |
+ --elements-max-string-length |
+ --bases | --bases-uniq |
+ --has-duplicates | --has-nested | --list |
+ --list-json | --list-chromosomes |
+ --archive-timestamp | --note |
+ --archive-version | --is-starch |
+ --signature | --verify-signature ]
+ <starch-file>
+
+ Modifiers
+ --------------------------------------------------------------------------
+ <chromosome> Optional. Either unarchives chromosome-
+ specific records from the starch archive
+ file or restricts action of operator to
+ chromosome (e.g., chr1, chrY, etc.).
+
+ Process Flags
+ --------------------------------------------------------------------------
+ --elements Show total element count for archive. If
+ <chromosome> is specified, the result
+ shows the element count for the
+ chromosome.
+
+ --elements-max-string-length Show the maximum string length over all
+ elements in <chromosome>, if specified.
+ If <chromosome> is not specified, the
+ maximum string length is shown over all
+ chromosomes.
- Process Flags:
-
- <chromosome> Optional. Either unarchives chromosome-specific records from the starch archive file or restricts action of operator to chromosome (e.g., chr1, chrY, etc.).
- --elements Show total element count for archive. If <chromosome> is specified, the result shows the element count for the chromosome.
--bases,
- --bases-uniq Show total and unique base counts, respectively, for archive. If <chromosome> is specified, the count is specific to the chromosome, if available.
- --has-duplicate, Show whether there is one or more duplicate elements in the specified chromosome, either as a numerical (1/0) or string (true/false) value. If no <chromosome> is specified, the value given indicates if there is one or more duplicate elements across all chromosome records.
- --has-duplicate-as-string
- --has-nested, Show whether there is one ore more nested elements in the specified chromosome, either as a numerical (1/0) or string (true/false) value. If no <chromosome> is specified, the value given indicates if there is one or more nested elements across all chromosome records.
- --has-nested-as-string
- --list List archive metadata (output is in text format). If chromosome is specified, the attributes of the given chromosome are shown.
- --list-json, List archive metadata (output is in JSON format)
- --list-json-no-trailing-newline
- --list-chr, List all or specified chromosome in starch archive (similar to "bedextract --list-chr"). If <chromosome> is specified but is not in the output list, nothing is returned.
- --list-chromosomes
+ --bases-uniq Show total and unique base counts,
+ respectively, for archive. If
+ <chromosome> is specified, the count is
+ specific to the chromosome, if available.
+
+ --has-duplicate-as-string,
+ --has-duplicate Show whether there is one or more
+ duplicate elements in the specified
+ chromosome, either as a numerical (1/0)
+ or string (true/false) value. If no
+ <chromosome> is specified, the value
+ given indicates if there is one or more
+ duplicate elements across all chromosome
+ records.
+
+ --has-nested-as-string,
+ --has-nested Show whether there is one ore more nested
+ elements in the specified chromosome,
+ either as a numerical (1/0) or string
+ (true/false) value. If no <chromosome> is
+ specified, the value given indicates if
+ there is one or more nested elements
+ across all chromosome records.
+
+ --list List archive metadata (output is in text
+ format). If chromosome is specified, the
+ attributes of the given chromosome are
+ shown.
+
+ --list-json,
+ --list-json-no-trailing-newline List archive metadata (output is in JSON
+ format)
+
+ --list-chr,
+ --list-chromosomes List all or specified chromosome in
+ starch archive (like "bedextract --list-
+ chr"). If <chromosome> is specified but
+ is not in the output list, nothing is
+ returned.
+
--note Show descriptive note, if available.
- --sha1-signature Show SHA1 signature of JSON-formatted metadata (Base64-encoded).
- --archive-timestamp Show archive creation timestamp (ISO 8601 format).
+
+ --signature Display the Base64-encoded SHA-1 data
+ integrity signature for specified
+ <chromosome>, or the signatures of the
+ metadata and all available chromosomes,
+ if the <chromosome> is unspecified.
+
+ --verify-signature Verify data integrity of specified
+ <chromosome>, or the integrity of all
+ available chromosomes, if the
+ <chromosome> is unspecified.
+
+ --archive-timestamp Show archive creation timestamp (ISO 8601
+ format).
+
--archive-type Show archive compression type.
+
--archive-version Show archive version.
- --is-starch Test if <starch-file> is a valid archive and print 0/1 (false/true) to standard output.
+
+ --is-starch Test if <starch-file> is a valid archive
+ and print 0/1 (false/true) to standard
+ output. Unstarch will also return a non-
+ zero error code if the input file is not
+ a valid archive.
+
--version Show binary version.
+
--help Show this usage message.
----------
@@ -107,7 +182,7 @@ Use the ``--list-json`` or ``--list`` options to export the archive metadata as
"creationTimestamp": "2014-05-01T14:09:29-0700",
"version": {
"major": 2,
- "minor": 1,
+ "minor": 2,
"revision": 0
},
"compressionFormat": 0
@@ -121,7 +196,9 @@ Use the ``--list-json`` or ``--list`` options to export the archive metadata as
"nonUniqueBaseCount": 549829,
"uniqueBaseCount": 548452,
"duplicateElementExists": false,
- "nestedElementExists": false
+ "nestedElementExists": false,
+ "signature": "XtnjojM1LyuMnZI4CIneSzgLI5Q="
+ "uncompressedLineMaxStringLength": 371
},
...
]
@@ -167,6 +244,20 @@ The ``--version`` option reports the Starch archive version. This value is diffe
Whole-file or per-chromosome attributes
---------------------------------------
+^^^^^^^^^^^^^^
+Data integrity
+^^^^^^^^^^^^^^
+
+For a specified chromosome, the ``--signature`` operator reports the very nearly unique "signature" or message digest generated from hashing the extracted, post-transform bytes within the chromosome stream with the `SHA-1 hash function <https://en.wikipedia.org/wiki/SHA-1>`_, followed with an encoding step with the `Base64 <https://en.wikipedia.org/wiki/Base64>`_ scheme to turn it into a human-readable string.
+
+If no chromosome is specified, this operator reports the encoded SHA-1 digests of the archive metadata and the signatures of each chromosome stream.
+
+Signatures can be used to compare chromosome streams between two or more archives.
+
+Further, use of the ``--verify-signature`` option with a chromosome name will compare the signature stored in the metadata (the "expected" signature) with an "observed" value generated from extracting the bytes of the chromosome record and hashing them, and then Base64-encoding the result.
+
+If the observed and expected signatures or digests are identical, this validates or verifies the integrity of the chromosome record. A mismatch would result in a non-zero exit state and suggest potential data corruption and the need for further investigation.
+
^^^^^^^^
Elements
^^^^^^^^
@@ -175,6 +266,8 @@ The ``--elements`` operator reports the number of BED elements that were compres
.. tip:: This option is equivalent to a ``wc -l`` (line count) operation performed on BED elements that match the given chromosome, but is much, much faster as data are precomputed and stored with the archive, retrieved from the metadata in O(1) time.
+The ``--elements-max-string-length`` operator reports the maximum string length of BED elements over the specified chromosome, or the maximum string length over all chromosomes, if no chromosome name is specified.
+
^^^^^
Bases
^^^^^
@@ -262,7 +355,26 @@ To show when the archive was created:
$ unstarch --archive-timestamp example.starch
2014-05-01T14:09:29-0700
-.. note:: Some option calls will not work with legacy v1.x or v2.0 archives. For instance, to get a result for nested or duplicate elements, you need to input a v2.1 archive. If you have a v1.x or v2.0 archive, use the :ref:`starchcat` utility to upgrade an older archive to a Starch v2.1 file, which will recalculate and make all current attributes available.
+To get the SHA-1 message digest, or "signature" of chromosome ``chr8``, use the ``--signature`` operator:
+
+::
+
+ $ unstarch chr8 --signature example.starch
+ nZI4CIneSzgLI5QXtnjojM1LyuM=
+
+The signature is written to the standard output stream.
+
+To verify the data integrity of the same chromosome, use ``--verify-signature``:
+
+::
+
+ $ unstarch chr8 --verify-signature example.starch
+ PROGRESS: Expected and observed data integrity signatures match for chromosome [chr8]
+
+Any output from ``--verify-signature`` is written to the standard error stream.
+
+
+.. note:: Some option calls will not work with legacy v1.x or v2.0 archives. For instance, to get a result for nested or duplicate elements, you need to input a v2.1 (or greater) archive. If you have a v1.x or v2.0 archive, use the :ref:`starchcat` utility to upgrade an older archive to a Starch v2.2 file, which will recalculate the metadata and make all current attributes available.
.. |--| unicode:: U+2013 .. en dash
.. |---| unicode:: U+2014 .. em dash, trimming surrounding whitespace
diff --git a/docs/content/reference/file-management/conversion/gtf2bed.rst b/docs/content/reference/file-management/conversion/gtf2bed.rst
index 58bb2bb..f0c2bf1 100644
--- a/docs/content/reference/file-management/conversion/gtf2bed.rst
+++ b/docs/content/reference/file-management/conversion/gtf2bed.rst
@@ -41,18 +41,18 @@ To demonstrate these scripts, we use a sample GTF input called ``foo.gtf`` (see
::
- chr20 protein_coding exon 9874841 9874841 . + . gene_id "ENSBTAG00000020601"; transcript_id "ENSBTAT0000002.4.20"; gene_name "ZNF366";
- chr20 protein_coding CDS 9873504 9874841 . + 0 gene_id "ENSBTAG00000020601"; transcript_id "ENSBTAT0000002.4.20"; gene_name "ZNF366";
- chr20 protein_coding exon 9877488 9877679 . + . gene_id "ENSBTAG00000020601"; transcript_id "ENSBTAT0000002.4.20";
+ chr20 protein_coding exon 9874841 9874841 . + . gene_id "ENSBTAG00000020601"; transcript_id "ENSBTAT0000002.4.26"; gene_name "ZNF366";
+ chr20 protein_coding CDS 9873504 9874841 . + 0 gene_id "ENSBTAG00000020601"; transcript_id "ENSBTAT0000002.4.26"; gene_name "ZNF366";
+ chr20 protein_coding exon 9877488 9877679 . + . gene_id "ENSBTAG00000020601"; transcript_id "ENSBTAT0000002.4.26";
We can convert it to sorted BED data in the following manner:
::
$ gtf2bed < foo.gtf
- chr20 9874840 9874841 ZNF366 . + protein_coding exon . gene_id "ENSBTAG00000020601"; transcript_id "ENSBTAT0000002.4.20"; gene_name "ZNF366"; zero_length_insertion "True";
- chr20 9873503 9874841 ZNF366 . + protein_coding CDS 0 gene_id "ENSBTAG00000020601"; transcript_id "ENSBTAT0000002.4.20"; gene_name "ZNF366";
- chr20 9877487 9877679 ENSBTAG00000020601 . + protein_coding exon . gene_id "ENSBTAG00000020601"; transcript_id "ENSBTAT0000002.4.20";
+ chr20 9874840 9874841 ZNF366 . + protein_coding exon . gene_id "ENSBTAG00000020601"; transcript_id "ENSBTAT0000002.4.26"; gene_name "ZNF366"; zero_length_insertion "True";
+ chr20 9873503 9874841 ZNF366 . + protein_coding CDS 0 gene_id "ENSBTAG00000020601"; transcript_id "ENSBTAT0000002.4.26"; gene_name "ZNF366";
+ chr20 9877487 9877679 ENSBTAG00000020601 . + protein_coding exon . gene_id "ENSBTAG00000020601"; transcript_id "ENSBTAT0000002.4.26";
.. tip:: After, say, performing set or statistical operations with :ref:`bedops`, :ref:`bedmap` etc., converting data back to GTF is accomplished through an ``awk`` statement that re-orders columns and shifts the coordinate index:
diff --git a/docs/content/reference/file-management/conversion/parallel_bam2bed.rst b/docs/content/reference/file-management/conversion/parallel_bam2bed.rst
index 6bc5366..3cca2b3 100644
--- a/docs/content/reference/file-management/conversion/parallel_bam2bed.rst
+++ b/docs/content/reference/file-management/conversion/parallel_bam2bed.rst
@@ -3,7 +3,7 @@
Parallel `bam2bed`
===================
-The ``bam2bed_sge`` and ``bam2bed_gnuParallel`` scripts use a Sun or Oracle Grid Engine (SGE/OGE) or `GNU Parallel <https://en.wikipedia.org/wiki/GNU_parallel>`_ job scheduler to parallelize the work of ``bam2bed``, which converts an **indexed**, 0-based, half-open ``[start-1, end)`` `Binary (Sequence) Alignment/Map <http://samtools.sourceforge.net/SAM1.pdf>`_ (BAM) file to a sorted, 0-based, half-open ``[start-1, end)`` UCSC BED dataset.
+The ``bam2bed_slurm``, ``bam2bed_sge``, and ``bam2bed_gnuParallel`` scripts use a SLURM, Sun or Oracle Grid Engine (SGE/OGE), or `GNU Parallel <https://en.wikipedia.org/wiki/GNU_parallel>`_ job scheduler, respectively, to parallelize the work of ``bam2bed``, which converts an **indexed**, 0-based, half-open ``[start-1, end)`` `Binary (Sequence) Alignment/Map <http://samtools.sourceforge.net/SAM1.pdf>`_ (BAM) file to a sorted, 0-based, half-open ``[start-1, end)`` UCSC BED dataset.
This script splits the indexed BAM file by chromosome name. Each chromosome of BAM records is converted to a BED-formatted dataset with ``bam2bed`` (via ``convert2bed``). Once all per-chromosome BED files are made, they are collated into one final BED file with a multiset union performed with :ref:`bedops --everything <bedops>`.
@@ -11,22 +11,22 @@ This script splits the indexed BAM file by chromosome name. Each chromosome of B
Dependencies
============
-This shell script is dependent upon a working computational grid that is managed with Sun Grid Engine 6.1u5 (or higher), or installation of GNU Parallel v20130922 or greater.
+This shell script is dependent upon a working computational grid that is managed with SLURM 16.05.0, Sun Grid Engine 6.1u5 (or higher), or installation of GNU Parallel v20130922 or greater.
======
Source
======
-The ``bam2bed_sge`` and ``bam2bed_gnuParallel`` conversion scripts are part of the binary and source downloads of BEDOPS. See the :ref:`Installation <installation>` documentation for more details.
+The ``bam2bed_slurm``, ``bam2bed_sge``, and ``bam2bed_gnuParallel`` conversion scripts are part of the binary and source downloads of BEDOPS. See the :ref:`Installation <installation>` documentation for more details.
=====
Usage
=====
-.. note:: Please review and edit the contents of the relevant script before use with your data, particularly if you use a Sun or Oracle Grid Engine environment and make use of the SGE version of this script. Customization may be required to match your SGE/OGE or GNU Parallel installation and environment, as well as the nature of your BAM data.
+.. note:: Please review and edit the contents of the relevant script before use with your data, particularly if you use a Sun or Oracle Grid Engine environment and make use of the SGE version of this script. Customization may be required to match your SLURM, SGE/OGE or GNU Parallel installation and environment, as well as the nature of your BAM data.
-At minimum, use of this script with an SGE/OGE computational cluster will require editing of the ``queue`` parameter, possible adjustments to ``qsub`` options, and may also require adjustments to paths to working BEDOPS and Samtools binaries.
+At minimum, use of this script with an SGE/OGE computational cluster will require editing of the ``queue`` parameter, possible adjustments to ``qsub`` options, and may also require adjustments to paths to working BEDOPS and Samtools binaries. If you use the `module` system, running `module add samtools` will take care of this dependency.
-You will also need to make sure your BAM data are indexed. There must be a second BAI file with the same name as the BAM file you wish to compress, located in the same working directory. If this index file is not present, the script will exit early with an error.
+You will also need to make sure your BAM data are indexed. There must be a second BAI file with the same prefix name as the BAM file you wish to compress, located in the same working directory. If this index file is not present, the script will exit early with an error.
You may also wish to review other parameters available with the ``bam2bed`` script, applying them in this script as needed (see the :ref:`bam2bed <bam2bed>` documentation for more details).
diff --git a/docs/content/reference/file-management/conversion/parallel_bam2starch.rst b/docs/content/reference/file-management/conversion/parallel_bam2starch.rst
index 4deb3d5..2165ff9 100644
--- a/docs/content/reference/file-management/conversion/parallel_bam2starch.rst
+++ b/docs/content/reference/file-management/conversion/parallel_bam2starch.rst
@@ -3,7 +3,7 @@
Parallel `bam2starch`
=====================
-The ``bam2starch_sge`` and ``bam2starch_gnuParallel`` scripts use a Sun or Oracle Grid Engine (SGE/OGE) or `GNU Parallel <https://en.wikipedia.org/wiki/GNU_parallel>`_ job scheduler to parallelize the work of ``bam2starch``, which converts an **indexed**, 0-based, half-open ``[start-1, end)`` `Binary (Sequence) Alignment/Map <http://samtools.sourceforge.net/SAM1.pdf>`_ (BAM) file to a sorted, 0-based, half-open ``[start-1, end)`` UCSC BED dataset, and thence converts this to a :ref:`Star [...]
+The ``bam2starch_slurm``, ``bam2starch_sge``, and ``bam2starch_gnuParallel`` scripts use a SLURM, Sun or Oracle Grid Engine (SGE/OGE), or `GNU Parallel <https://en.wikipedia.org/wiki/GNU_parallel>`_ job scheduler, respectively, to parallelize the work of ``bam2starch``, which converts an **indexed**, 0-based, half-open ``[start-1, end)`` `Binary (Sequence) Alignment/Map <http://samtools.sourceforge.net/SAM1.pdf>`_ (BAM) file to a sorted, 0-based, half-open ``[start-1, end)`` UCSC BED dat [...]
This script splits the indexed BAM file by chromosome name. Each chromosome of BAM records is converted to a :ref:`Starch-formatted <starch_specification>` archive with ``bam2starch`` (via ``convert2bed``). Once all per-chromosome archives are made, they are collated into one final Starch archive with :ref:`starchcat <starchcat>`.
@@ -13,22 +13,22 @@ This script splits the indexed BAM file by chromosome name. Each chromosome of B
Dependencies
============
-This shell script is dependent upon a working computational grid that is managed with Sun Grid Engine 6.1u5 (or higher), or installation of GNU Parallel v20130922 or greater.
+This shell script is dependent upon a working computational grid that is managed with SLURM 16.05.0, Sun Grid Engine 6.1u5 (or higher), or installation of GNU Parallel v20130922 or greater.
======
Source
======
-The ``bam2starch_sge`` and ``bam2starch_gnuParallel`` conversion scripts are part of the binary and source downloads of BEDOPS. See the :ref:`Installation <installation>` documentation for more details.
+The ``bam2starch_slurm``, ``bam2starch_sge``, and ``bam2starch_gnuParallel`` conversion scripts are part of the binary and source downloads of BEDOPS. See the :ref:`Installation <installation>` documentation for more details.
=====
Usage
=====
-.. note:: Please review and edit the contents of this script before use with your data. Customization may be required to match your SGE/OGE or GNU Parallel installation and environment, as well as the nature of your BAM data.
+.. note:: Please review and edit the contents of this script before use with your data. Customization may be required to match your SLURM, SGE/OGE, or GNU Parallel installation and environment, as well as the nature of your BAM data.
At minimum, use of this script with an SGE/OGE computational cluster will require editing of the ``queue`` parameter, possible adjustments to ``qsub`` options, and may require adjustments to paths to working BEDOPS binaries.
-You will also need to make sure your BAM data are indexed. There must be a second BAI file with the same name as the BAM file you wish to compress, located in the same working directory. If this index file is not present, the script will exit early with an error.
+You will also need to make sure your BAM data are indexed. There must be a second BAI file with the same prefix name as the BAM file you wish to compress, located in the same working directory. If this index file is not present, the script will exit early with an error.
You may also wish to review other parameters available with the ``bam2starch`` script, applying them in this script as needed (see the :ref:`bam2bed <bam2bed>` documentation for more details).
diff --git a/docs/content/reference/file-management/conversion/psl2bed.rst b/docs/content/reference/file-management/conversion/psl2bed.rst
index 9c0e318..0e4763d 100644
--- a/docs/content/reference/file-management/conversion/psl2bed.rst
+++ b/docs/content/reference/file-management/conversion/psl2bed.rst
@@ -57,7 +57,7 @@ To demonstrate these scripts, we use a sample PSL input called ``foo.psl`` (see
30 0 0 0 0 0 0 0 + foo 50 14 44 chr6 171115067 127685756 127685786 1 30, 14, 127685756,
30 0 0 0 0 0 0 0 + foo 50 14 44 chr6 171115067 93161871 93161901 1 30, 14, 93161871,
31 0 0 0 0 0 0 0 + foo 50 13 44 chr5 180915260 119897315 119897346 1 31, 13, 119897315,
- 30 0 0 0 0 0 0 0 + foo 50 14 44 chr5 180915260 123254725 1232.4.205 1 30, 14, 123254725,
+ 30 0 0 0 0 0 0 0 + foo 50 14 44 chr5 180915260 1232.4.26 1232.4.265 1 30, 14, 1232.4.26,
...
We can convert it to sorted BED data in the following manner:
diff --git a/docs/content/reference/file-management/conversion/vcf2bed.rst b/docs/content/reference/file-management/conversion/vcf2bed.rst
index bc77780..342f393 100644
--- a/docs/content/reference/file-management/conversion/vcf2bed.rst
+++ b/docs/content/reference/file-management/conversion/vcf2bed.rst
@@ -54,7 +54,7 @@ In addition, using any of these three custom options automatically results in pr
::
#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT NA00001 NA00002 NA00003
- 20 12.4.207 microsat1 GTC G,GTCT 50 PASS NS=3;DP=9;AA=G GT:GQ:DP 0/1:35:4 0/2:17:2 1/1:40:3
+ 20 12.4.267 microsat1 GTC G,GTCT 50 PASS NS=3;DP=9;AA=G GT:GQ:DP 0/1:35:4 0/2:17:2 1/1:40:3
When using ``--snvs``, ``--insertions`` or ``--deletions``, this record is split into two distinct BED records and filtered depending on which custom option was chosen. The ``--insertions`` option would only export the single-base position of the insertion in this mixed variant, while ``--deletions`` would show the deletion.
diff --git a/docs/content/reference/file-management/sorting/sort-bed.rst b/docs/content/reference/file-management/sorting/sort-bed.rst
index 65fa34a..2ae6be6 100644
--- a/docs/content/reference/file-management/sorting/sort-bed.rst
+++ b/docs/content/reference/file-management/sorting/sort-bed.rst
@@ -5,10 +5,18 @@
The ``sort-bed`` utility sorts BED files of any size, even larger than system memory. BED files that are in lexicographic-chromosome order allow BEDOPS utilities to work efficiently with data from any species without software modifications. Further, sorted files can be traversed very quickly.
-Sorted BED order is defined first by lexicographic chromosome order, then ascending integer start coordinate order, and finally by ascending integer end coordinate order.
+Sorted BED order is defined first by lexicographic chromosome order, then ascending integer start coordinate order, and finally by ascending integer end coordinate order. To make the sort order unambiguous, a lexicographical sort is applied on fourth and subsequent columns, where present in the input BED dataset.
Other utilities in the BEDOPS suite require data in sorted order as described. You only need to sort once: BEDOPS utilities all read and write data in sorted order.
+====================================
+Migrating older BED and Starch files
+====================================
+
+The utility ``update-sort-bed-migrate-candidates`` recursively locates BED and pre-v2.4.20 Starch files in the specified parent directory, tests if they require re-sorting to conform to the updated, post-v2.4.20 'sort-bed' order, and offers actions to log candidate files, or immediately apply a resort action that is performed locally or via a SLURM-managed cluster.
+
+The convenience utilities ``update-sort-bed-slurm`` and ``update-sort-bed-starch-slurm`` update the sort order of BED or Starch files sorted with pre-v2.4.20 ``sort-bed`` via a SLURM-based cluster. See ``update-sort-bed-slurm --help`` or ``update-sort-bed-starch-slurm --help`` for more details. These utilities can be used standalone or in conjunction with the ``update-sort-bed-migrate-candidates`` utility.
+
==================
Inputs and outputs
==================
@@ -17,13 +25,13 @@ Inputs and outputs
Input
-----
-The :ref:`sort-bed` utility requires one or more three-column BED file(s). Support for common headers (such as `UCSC BED track headers <http://genome.ucsc.edu/FAQ/FAQformat.html#format1>`_) is included, although headers will be stripped from the output.
+The ``sort-bed`` utility requires one or more three-column BED file(s). Support for common headers (such as `UCSC BED track headers <http://genome.ucsc.edu/FAQ/FAQformat.html#format1>`_) is included, although headers will be stripped from the output.
------
Output
------
-The :ref:`sort-bed` utility sends lexicographically-sorted BED data to standard output, which can be redirected to a file or piped to other utilities, including core BEDOPS utilities like :ref:`bedops` and :ref:`bedmap`.
+The ``sort-bed`` utility sends sorted BED data to standard output, which can be redirected to a file or piped to other utilities, including core BEDOPS utilities like :ref:`bedops` and :ref:`bedmap`. Sort order is defined by a lexicographical sort on chromosome name, a numerical sort on start coordinates, a numerical sort on stop coordinates where there are start matches, and finally a lexicographical sort on the remainder of the BED element (if additional columns are present).
=====
Usage
@@ -35,7 +43,7 @@ The ``--help`` option is fairly basic, but describes the usage:
sort-bed
citation: http://bioinformatics.oxfordjournals.org/content/28/14/1919.abstract
- version: 2.4.20
+ version: 2.4.26
authors: Scott Kuehn
USAGE: sort-bed [--help] [--version] [--check-sort] [--max-mem <val>] [--tmpdir <path>] <file1.bed> <file2.bed> <...>
@@ -46,19 +54,19 @@ The ``--help`` option is fairly basic, but describes the usage:
<val> for --max-mem may be 8G, 8000M, or 8000000000 to specify 8 GB of memory.
--tmpdir is useful only with --max-mem.
-A simple example of using :ref:`sort-bed` would be:
+A simple example of using ``sort-bed`` would be:
::
$ sort-bed unsortedData.bed > sortedData.bed
-The :ref:`sort-bed` program efficiently sorts BED inputs. By default, all input records are read into system memory and sorted. If your BED dataset is larger than available system memory, use the ``--max-mem`` option to limit the amount of memory :ref:`sort-bed` uses to do its work:
+The ``sort-bed`` program efficiently sorts BED inputs. By default, all input records are read into system memory and sorted. If your BED dataset is larger than available system memory, use the ``--max-mem`` option to limit the amount of memory ``sort-bed`` uses to do its work:
::
$ sort-bed --max-mem 2G reallyHugeUnsortedData.bed > reallyHugeSortedData.bed
-This option allows :ref:`sort-bed` to scale to input of any size.
+This option allows ``sort-bed`` to scale to input of any size.
The ``--tmpdir`` option allows specification of an alternative temporary directory, when used in conjunction with ``--max-mem`` option. This is useful if the host operating system’s standard temporary directory (*e.g.*, ``/tmp`` on Linux or OS X) does not have sufficient space to hold intermediate results.
diff --git a/docs/content/reference/set-operations/bedextract.rst b/docs/content/reference/set-operations/bedextract.rst
index f21845e..625c972 100644
--- a/docs/content/reference/set-operations/bedextract.rst
+++ b/docs/content/reference/set-operations/bedextract.rst
@@ -139,7 +139,7 @@ Three criteria make the use of :ref:`bedextract` in this mode very successful in
| bedops --element-of 1 - Target \
> answer.bed
-.. note:: You may change the final overlap criterion to the `bedops --element-of` as you see fit for your final answer.
+.. note:: You may change the final overlap criterion to the `bedops --element-of` as you see fit for your final answer. Adding --range 10, for example, to `bedops -m Query > Query-Index` can often greatly reduce the size of Query-Index without much of an increase in the running time of this approach.
.. _bedextract_nested_elements:
diff --git a/docs/content/reference/set-operations/bedops.rst b/docs/content/reference/set-operations/bedops.rst
index 24ae1c4..c1c3abd 100644
--- a/docs/content/reference/set-operations/bedops.rst
+++ b/docs/content/reference/set-operations/bedops.rst
@@ -53,7 +53,7 @@ The ``--help`` option describes the set operation and other options available to
bedops
citation: http://bioinformatics.oxfordjournals.org/content/28/14/1919.abstract
- version: 2.4.20
+ version: 2.4.26
authors: Shane Neph & Scott Kuehn
USAGE: bedops [process-flags] <operation> <File(s)>*
diff --git a/docs/content/reference/set-operations/closest-features.rst b/docs/content/reference/set-operations/closest-features.rst
index 25b441f..6ea27ef 100644
--- a/docs/content/reference/set-operations/closest-features.rst
+++ b/docs/content/reference/set-operations/closest-features.rst
@@ -45,7 +45,7 @@ The ``--help`` option describes the various operations and options available to
closest-features
citation: http://bioinformatics.oxfordjournals.org/content/28/14/1919.abstract
- version: 2.4.20
+ version: 2.4.26
authors: Shane Neph & Scott Kuehn
USAGE: closest-features [Process-Flags] <input-file> <query-file>
diff --git a/docs/content/reference/statistics/bedmap.rst b/docs/content/reference/statistics/bedmap.rst
index 5cfc374..7ea7fad 100644
--- a/docs/content/reference/statistics/bedmap.rst
+++ b/docs/content/reference/statistics/bedmap.rst
@@ -48,7 +48,7 @@ The ``--help`` option describes the various mapping and analytical operations an
bedmap
citation: http://bioinformatics.oxfordjournals.org/content/28/14/1919.abstract
- version: 2.4.20
+ version: 2.4.26
authors: Shane Neph & Scott Kuehn
USAGE: bedmap [process-flags] [overlap-option] <operation(s)...> <ref-file> [map-file]
@@ -106,17 +106,21 @@ The ``--help`` option describes the various mapping and analytical operations an
--mad <mult=1> The median absolute deviation of overlapping elements in <map-file>.
Multiply mad score by <mult>. 0 < mult, and mult is 1 by default.
--max The highest score from overlapping elements in <map-file>.
- --max-element An element with the highest score from overlapping elements in <map-file>.
+ --max-element A (non-random) highest-scoring and overlapping element in <map-file>.
+ --max-element-rand A random highest-scoring and overlapping element in <map-file>.
--mean The average score from overlapping elements in <map-file>.
--median The median score from overlapping elements in <map-file>.
--min The lowest score from overlapping elements in <map-file>.
- --min-element An element with the lowest score from overlapping elements in <map-file>.
+ --min-element A (non-random) lowest-scoring and overlapping element in <map-file>.
+ --min-element-rand A random lowest-scoring and overlapping element in <map-file>.
--stdev The square root of the result of --variance.
--sum Accumulated scores from overlapping elements in <map-file>.
--tmean <low> <hi> The mean score from overlapping elements in <map-file>, after
ignoring the bottom <low> and top <hi> fractions of those scores.
0 <= low <= 1. 0 <= hi <= 1. low+hi <= 1.
--variance The variance of scores from overlapping elements in <map-file>.
+ --wmean Weighted mean, scaled in proportion to the coverage of the <ref-file>
+ element by each overlapping <map-file> element.
----------
NON-SCORE:
@@ -266,6 +270,7 @@ The variety of score operators include common statistical measures:
* `mean <http://en.wikipedia.org/wiki/Expected_value>`_ (``--mean``)
* `trimmed mean <http://en.wikipedia.org/wiki/Truncated_mean>`_ (``--tmean``)
+* `weighted mean <http://en.wikipedia.org/wiki/Weighted_arithmetic_mean>`_ (``--wmean``)
* `standard deviation <http://en.wikipedia.org/wiki/Standard_deviation>`_ (``--stdev``)
* `variance <http://en.wikipedia.org/wiki/Variance>`_ (``--variance``)
* `coefficient of variance <http://en.wikipedia.org/wiki/Coefficient_of_variation>`_ (``--cv``)
@@ -371,7 +376,9 @@ Likewise, we can repeat this operation, but look for the lowest scoring elements
chr21 33031400 33031800 ref-2|chr21 33031525 33031545 map-19 13
chr21 33031900 33032000 ref-3|chr21 33031985 33032005 map-42 138
-.. note:: Where there are ties in score values, there is no guarantee about which tied element will be chosen. In this case, the ``--echo-map`` operator can be used to manually examine the full list of elements and apply different logic.
+.. note:: Where there are ties in score values, using ``--max-element`` or ``--min-element`` now selects the lexicographically smallest element amongst the set of tied elements. This generally means that the first element in the lexicographic ordering of the ID fields (fourth column) will determine the selection.
+
+ A random selection process was used for ``--max-element`` and ``--min-element`` in versions 2.4.20 and previous. If you wish to randomly sample a maximum- or minimum-scoring element from amongst tied elements (say, to reproduce the procedure of prior analyses), you may use the ``--max-element-rand`` or ``--min-element-rand`` options, respectively.
We can also perform multiple score operations, which are summarized on one line, *e.g.*, to show the mean, standard deviation, and minimum and maximum signal over each ``Reference`` element, we simply add the requisite options in series:
@@ -413,9 +420,9 @@ As an example of using the ``--echo-map-id`` operator in a biological context, w
::
chr1 4534161 4534177 -V_GRE_C 4.20586e-06 - CGTACACACAGTTCTT
- chr1 4534192.4.204205 -V_STAT_Q6 2.21622e-06 - AGCACTTCTGGGA
+ chr1 4534192.4.264205 -V_STAT_Q6 2.21622e-06 - AGCACTTCTGGGA
chr1 4534209 4534223 +V_HNF4_Q6_01 6.93604e-06 + GGACCAGAGTCCAC
- chr1 4962522.4.202540 -V_GCNF_01 9.4497e-06 - CCCAAGGTCAAGATAAAG
+ chr1 4962522.4.262540 -V_GCNF_01 9.4497e-06 - CCCAAGGTCAAGATAAAG
chr1 4962529 4962539 +V_NUR77_Q5 8.43564e-06 + TTGACCTTGG
...
@@ -667,9 +674,9 @@ To demonstrate their use, we revisit the ``Motifs`` dataset, which includes *p*-
::
chr1 4534161 4534177 -V_GRE_C 4.20586e-06 - CGTACACACAGTTCTT
- chr1 4534192.4.204205 -V_STAT_Q6 2.21622e-06 - AGCACTTCTGGGA
+ chr1 4534192.4.264205 -V_STAT_Q6 2.21622e-06 - AGCACTTCTGGGA
chr1 4534209 4534223 +V_HNF4_Q6_01 6.93604e-06 + GGACCAGAGTCCAC
- chr1 4962522.4.202540 -V_GCNF_01 9.4497e-06 - CCCAAGGTCAAGATAAAG
+ chr1 4962522.4.262540 -V_GCNF_01 9.4497e-06 - CCCAAGGTCAAGATAAAG
chr1 4962529 4962539 +V_NUR77_Q5 8.43564e-06 + TTGACCTTGG
...
diff --git a/docs/content/release.rst b/docs/content/release.rst
index 1f78bc0..812d498 100644
--- a/docs/content/release.rst
+++ b/docs/content/release.rst
@@ -15,7 +15,7 @@ Preparing a major, minor or maintenance release of BEDOPS from a development bra
a. Close out open documentation or feature issues, making necessary pushes to the current development branch.
- b. If any issues can't be closed out, rename the assigned version tag to the next anticipated release version (*e.g.*, *v2.4.20* to *v2p5p0*, etc.)
+ b. If any issues can't be closed out, rename the assigned version tag to the next anticipated release version (*e.g.*, *v2.4.26* to *v2p5p0*, etc.)
2. Pull the most recent commit for the development branch to a local folder on build hosts (Linux with sufficiently old kernel, current OS X, etc.).
@@ -44,7 +44,7 @@ Preparing a major, minor or maintenance release of BEDOPS from a development bra
BEDOPS.X.Y.Z.pkg.zip
- The *X.Y.Z* scheme should follow the development branch name, *e.g.* 2.4.20, etc.
+ The *X.Y.Z* scheme should follow the development branch name, *e.g.* 2.4.26, etc.
3. Collect tarballs and zipped Installer in one location for later addition with web browser, via BEDOPS Github web site.
@@ -63,7 +63,7 @@ Release
Ideally, whatever steps are used to merge the development branch into the master branch should preserve the overall commit history.
- As before, the *X.Y.Z* scheme should follow the development branch name, *e.g.* 2.4.20, etc.
+ As before, the *X.Y.Z* scheme should follow the development branch name, *e.g.* 2.4.26, etc.
2. Add a `new release <https://github.com/bedops/bedops/releases/new>`_ via the Github site. Or click on the `Draft a new release <https://github.com/bedops/bedops/releases>`_ button from the Github Releases page.
@@ -94,7 +94,7 @@ Release
### Mac OS X
**BEDOPS.X.Y.Z.pkg.zip**
- This package of BEDOPS vX.Y.Z binaries is a digitally-signed installer for OS X (10.7 - 10.11) running on Intel-based Macs.
+ This package of BEDOPS vX.Y.Z binaries is a digitally-signed installer for OS X (10.7 - 10.12) running on Intel-based Macs.
For installation instructions, please read [§2.1.2. Mac OS X] (http://bedops.readthedocs.io/en/latest/content/installation.html#mac-os-x) of the BEDOPS Installation document.
diff --git a/docs/content/revision-history.rst b/docs/content/revision-history.rst
index 0195785..3d70388 100644
--- a/docs/content/revision-history.rst
+++ b/docs/content/revision-history.rst
@@ -12,22 +12,232 @@ Current version
===============
-------
-v2.4.20
+v2.4.26
-------
-Released: **July 27, 2016**
+Released: **March 14, 2017**
-* :ref:`convert2bed <convert2bed>`
+* :ref:`starchstrip <starchstrip>`
- * Increased memory allocation for maximum number of per-read CIGAR operations in BAM and SAM conversion to help improve stability. Thanks to Adam Freedman for the report!
+ * New utility to efficiently filter a Starch archive, including or excluding records by specified chromosome names, without doing expensive extraction and recompression. This follows up on `internal discussion <https://stamlab.slack.com/archives/bedops/p1487878245000103>`_ on the Altius Slack channel.
+
+* :ref:`starch-diff <starch_diff>`
+
+ * Fixed testing logic in :code:`starch-diff` for certain archives. Thanks to Shane Neph for the report.
+
+* :ref:`starchcat <starchcat>`
+
+ * Fixed possible condition where too many variables on the stack can cause a stack overload on some platforms, leading to a fatal segmentation fault. Improved logic for updating v2.1 to v2.2 Starch archives.
+
+* Starch C++ API
+
+ * Patched gzip-backed Starch archive extraction issue. Thanks to Matt Maurano for the bug report.
+
+* :ref:`update-sort-bed-migrate-candidates <sort-bed>`
+
+ * Added detailed logging via :code:`--debug` option.
+
+ * Added :code:`--bedops-root-dir` option to allow specifying where all BEDOPS binaries are stored. This setting can be overruled on a per-binary basis by adding :code:`--bedextract-path`, :code:`--sort-bed-path`, etc.
+
+ * Added :code:`--non-recursive-search` option to restrict search for BED and Starch candidates to the top-level of the specified parent directory :code:`--parent-dir` option.
+
+ * Further simplification and customization of parameters sent to :code:`update-sort-bed-slurm` and :code:`update-sort-bed-starch-slurm` cluster scripts, as well as logging and variable name improvements to those two scripts.
+
+ * Thanks again to Matt Maurano for ongoing feedback and suggestions on functionality and fixes.
+
+* :ref:`gtf2bed <gtf2bed>`
- * Improved reliability of gene ID parsing from GTF input, where ``gene_id`` field may be positioned at start, middle, or end of attributes string, or may be empty. Thanks to blaiseli for the report!
+ * Resolved segmentation fault with certain inputs, in follow-up to `this BEDOPS Forum post <http://bedops.uwencode.org/forum/index.php?topic=136.0>`_. Thanks to zebasilio for the report and feedback.
=================
Previous versions
=================
-------
+v2.4.25
+-------
+
+Released: **February 15, 2017**
+
+* :ref:`convert2bed <convert2bed>`
+
+ * Patch for RepeatMasker inputs with blank lines that have no spaces. This follows up on `Issue 173 <https://github.com/bedops/bedops/issues/173>`_. Thanks to saketkc for the bug report.
+
+* :ref:`update-sort-bed-migrate-candidates <sort-bed>`
+
+ The :code:`update-sort-bed-migrate-candidates` utility recursively searches into the specified directory for BED and Starch files which fail a :code:`sort-bed --check-sort` test. Those files which fail this test can have their paths written to a text file for further downstream processing, or the end user can decide to apply an immediate resort on those files, either locally or via a SLURM-managed cluster. Grateful thanks to Matt Maurano for input and testing.
+
+ See :code:`update-sort-bed-migrate-candidates --help` for more information, or review the :ref:`sort-bed <sort-bed>` documentation.
+
+* :ref:`update-sort-bed-starch-slurm <sort-bed>`
+
+ This is an adjunct to the :code:`update-sort-bed-slurm` utility, which resorts the provided Starch file and writes a new file. (The :code:`update-sort-bed-slurm` utility only takes in BED files as input and writes BED as output.)
+
+-------
+v2.4.24
+-------
+
+Released: **February 6, 2017**
+
+* :ref:`starch-diff <starch_diff>`
+
+ * The :code:`starch-diff` utility compares signatures of two or more v2.2+ Starch archives. This tool tests all chromosomes or one specified chromosome. It returns a zero exit code, if the signature(s) are identical, or a non-zero error exit code, if one or more signature(s) are dissimilar.
+
+* :ref:`update-sort-bed-slurm <sort-bed>`
+
+ * The :code:`update-sort-bed-slurm` convenience utility provides a parallelized update of the sort order on BED files sorted with pre-v2.4.20 sort-bed, for users with a SLURM job scheduler and associated cluster. See :code:`update-sort-bed-slurm --help` for more details.
+
+* :ref:`convert2bed <convert2bed>`
+
+ * Patched a memory leak in VCF conversion. Thanks to ehsueh for the bug report.
+
+-------
+v2.4.23
+-------
+
+Released: **January 30, 2017**
+
+* :ref:`unstarch <unstarch>`
+
+ * Fixed bug where missing signature from pre-v2.2 Starch archives would cause a fatal metadata error. Thanks to Shane Neph and Eric Rynes for the bug report.
+
+ * Improved logic reporting signature mismatches when input v2.2 archive lacks signature (*e.g.*, for a v2.2 archive made with :code:`--omit-signature`).
+
+* :ref:`starch <starch>` and :ref:`starchcat <starchcat>`
+
+ * Added :code:`--omit-signature` option to compress without creating a per-chromosome data integrity signature. While this reduces compression time, this eliminates the verification benefits of the data integrity signature.
+
+-------
+v2.4.22
+-------
+
+Released: **January 25, 2017**
+
+* :ref:`convert2bed <convert2bed>`
+
+ * Fixed heap corruption in GFF conversion. Thanks to J. Miguel Mendez (ObjectiveTruth) for the bug report.
+
+-------
+v2.4.21
+-------
+
+Released: **January 23, 2017**
+
+* :ref:`bedmap <bedmap>`
+
+ * New :code:`--wmean` operation offers a weighted mean calculation. The "weight" is derived from the proportion of the reference element covered by overlapping map elements: *i.e.*, a map element that covers more of the reference element has its signal given a larger weight or greater impact than another map element with a shorter overlap.
+
+ * Measurement values in :code:`bedmap` did not allow :code:`+` in the exponent (both :code:`-` worked and no :code:`+` for a positive value. Similarly, out in front of the number, :code:`+` was previously not allowed. Shane Neph posted the report and the fix.
+
+ * The :code:`--min-element` and :code:`--max-element` operations in :ref:`bedmap <bedmap>` now process elements in unambiguous order. Former behavior is moved to the operations :code:`--min-element-rand` and :code:`--max-element-rand`, respectively.
+
+ * Fixed issue with use of :code:`--echo-overlap-size` with :code:`--multidelim` (cf. `Issue 165 <https://github.com/bedops/bedops/issues/165>`_). Shane Neph posted the fix. Thanks to Jeff Vierstra for the bug report!
+
+* :ref:`bedops <bedops>`
+
+ * Fixed issue with :code:`--chop` where complement operation could potentially be included. Shane Neph posted the fix.
+
+ * The :code:`bedops --everything` or :code:`bedops -u` (union) operation now writes elements to standard output in unambiguous sort order. If any data are contained in fourth or subsequent fields, a lexicographical sort on that data is applied for resolving order of interval matches.
+
+* :ref:`sort-bed <sort-bed>`
+
+ * Improved sort times from replacing quicksort (:code:`std::qsort`) with inlined C++ :code:`std::sort`.
+
+ * Sorting of BED input now leads to unambiguous result when two or more elements have the same genomic interval (chromosome name and start and stop position), but different content in remaining columns (ID, score, etc.).
+
+ Formerly, elements with the same genomic interval that have different content in fourth and subsequent columns could be printed in a non-consistent ordering on repeated sorts. A deterministic sort order facilitates the use of data integrity functions on sorted BED and Starch data.
+
+* :ref:`starchcluster <starchcluster>`
+
+ * A SLURM-ready version of the :code:`starchcluster` script was added to help SLURM job scheduler users with parallelizing the creation of Starch archives.
+
+* Parallel :ref:`bam2bed <parallel_bam2bed>` and :ref:`bam2starch <parallel_bam2starch>`
+
+ * SLURM-ready versions of these scripts were added to help parallelize the conversion of BAM to BED files (:code:`bam2bed_slurm`) or to Starch archives (:code:`bam2starch_slurm`).
+
+* :ref:`unstarch <unstarch>`
+
+ * Added :code:`--signature` option to report the Base64-encoded SHA-1 data integrity signature of the Starch-transformed bytes of a specified chromosome, or to report the signature of the metadata string as well as the signatures of all chromosomes, if unspecified.
+
+ * Added :code:`--verify-signature` option to compare the "expected" Base64-encoded SHA-1 data integrity signature stored within the archive's metadata with the "observed" data integrity signature generated from extracting the specified chromosome.
+
+ If the observed and expected signatures differ, then this suggests that the chromosome record may be corrupted in some way; :code:`unstarch` will exit with a non-zero error code. If the signatures agree, the archive data should be intact and `unstarch` will exit with a helpful notice and a zero error code.
+
+ If no chromosome is specified, :code:`unstarch` will loop through all chromosomes in the archive metadata, comparing observed and expected values for each chromosome record. Upon completion, error and progress messages will be reported to the standard error stream, and :code:`unstarch` will exit with a zero error code, if all signatures match, or a non-zero exit state, if one or more signatures do not agree.
+
+ * The output from the :code:`--list` option includes a :code:`signature` column to report the data integrity signature of all Starch-transformed chromosome data.
+
+ * The output from the :code:`--list-json` option includes a :code:`signature` key in each chromosome record in the archive metadata, reporting the same information.
+
+ * The :code:`--is-starch` option now quits with a non-zero exit code, if the specified input file is not a Starch archive.
+
+ * The :code:`--elements-max-string-length` option reports the length of the longest string within the specified chromosome, or the longest string over all chromosomes (if no chromosome name is specified).
+
+* :ref:`starch <starch>`
+
+ * Added :code:`--report-progress=N` option to (optionally) report compression of the Nth element of the current chromosome to standard error stream.
+
+ * As a chromosome is compressed, the input Starch-transform bytes are continually run through a SHA-1 hash function. The resulting data integrity signature is stored as a Base64-encoded string in the output archive's metadata. Signatures can be compared between and within archives to help better ensure the data integrity of the archive.
+
+ * Fixed :code:`--header` transform bug reported in `Issue 161 <https://github.com/bedops/bedops/issues/161>`_. Thanks to Shane Neph for the bug report!
+
+ * Added chromosome name and "remainder" order tests to :code:`STARCH2_transformHeaderlessBEDInput` and :code:`STARCH2_transformHeaderedBEDInput` functions.
+
+ Compression with :code:`starch` ends with a fatal error, should any of the following comparison tests fail:
+
+ 1. The chromosome names are not lexicographically ordered (*e.g.*, :code:`chr1` records coming after :code:`chr2` records indicates the data are not correctly sorted).
+
+ 2. The start position of an input element is less than the start position of a previous input element on the same chromosome (*e.g.*, :code:`chr1:1000-1234` coming after :code:`chr1:2000-2345` is not correctly sorted).
+
+ 3. The stop positions of two or more input elements are not in ascending order when their start positions are equal (*e.g.*, :code:`chr1:1000-1234` coming after :code:`chr1:1000-2345` is not correctly sorted).
+
+ 4. The start and stop positions of two or more input elements are equivalent, and their "remainders" (fourth and subsequent columns) are not in ascending order (*e.g.*, :code:`chr1:1000-1234:id-0` coming after :code:`chr1:1000-1234:id-1` is not correctly sorted).
+
+ If the sort order of the input data is unknown or uncertain, simply use :code:`sort-bed` to generate the correct ordering and pipe the output from that to :code:`starch`, *e.g.* :code:`$ cat elements.bed | sort-bed - | starch - > elements.starch`.
+
+* :ref:`starchcat <starchcat>`
+
+ * Added :code:`--report-progress=N` option to (optionally) report compression of the *N* th element of the current chromosome to standard error stream.
+
+ * As in :code:`starch`, at the conclusion of compressing a chromosome made from one or more input Starch archives, the input Starch-transform bytes are continually run through a SHA-1 hash function. The resulting data integrity signature is stored as a Base64-encoded string in the chromosome's entry in the new archive's metadata.
+
+ * As in :code:`starch`, if data should need to be extracted and recompressed, the output is written so that the order is unambiguous: ascending lexicographic ordering on chromosome names, numerical ordering on start positions, the same ordering on stop positions where start positions match, and ascending lexicographic ordering on the remainder of the BED element (fourth and subsequent columns, where present).
+
+* :ref:`convert2bed <convert2bed>`
+
+ * Improvements in support for BAM/SAM inputs with larger-sized reads, as would come from alignments made from data collected from third-generation sequencers. Simulated read datasets were generated using `SimLoRD <https://bitbucket.org/genomeinformatics/simlord/>`_. Tests have been performed on simulated hg19 data up to 100kb read lengths.
+
+ Improvements allow:
+
+ * conversion of dynamic number of CIGAR operations (up to system memory)
+
+ * conversion of dynamically-sized read fields (up to system memory and inter-thread buffer allocations)
+
+ These patches follow up on bug reports in `Issue 157 <https://github.com/bedops/bedops/issues/157>`_.
+
+ * Improvements in support for VCF inputs, to allow aribtrary-sized fields (up to system memory and inter-thread buffer allocations), which should reduce or eliminate segmentation faults from buffer overruns on fields larger than former stack defaults.
+
+ * Improvements in support for GFF inputs, to allow aribtrary-sized fields (up to system memory and inter-thread buffer allocations), which should reduce or eliminate segmentation faults from buffer overruns on fields larger than former stack defaults.
+
+ * Improvements in support for GTF inputs, to allow aribtrary-sized fields (up to system memory and inter-thread buffer allocations), which should reduce or eliminate segmentation faults from buffer overruns on fields larger than former stack defaults.
+
+* Testing
+
+ * Our use of Travis CI to automate testing of builds now includes Clang on `their OS X environment <https://docs.travis-ci.com/user/osx-ci-environment/>`_.
+
+-------
+v2.4.20
+-------
+
+Released: **July 27, 2016**
+
+* :ref:`convert2bed <convert2bed>`
+
+ * Increased memory allocation for maximum number of per-read CIGAR operations in BAM and SAM conversion to help improve stability. Thanks to Adam Freedman for the report!
+
+ * Improved reliability of gene ID parsing from GTF input, where :code:`gene_id` field may be positioned at start, middle, or end of attributes string, or may be empty. Thanks to blaiseli for the report!
+
+-------
v2.4.19
-------
@@ -35,11 +245,11 @@ Released: **May 9, 2016**
* :ref:`convert2bed <convert2bed>`
- * Fixed bug in BAM and SAM parallel conversion scripts (``*_gnuParallel`` and ``*_sge``) with inputs containing chromosome names without ``chr`` prefix. Thanks to Eric Haugen for the bug report!
+ * Fixed bug in BAM and SAM parallel conversion scripts (:code:`*_gnuParallel` and :code:`*_sge`) with inputs containing chromosome names without :code:`chr` prefix. Thanks to Eric Haugen for the bug report!
* Starch C++ API
- * Fixed bug with extraction of bzip2- and gzip-backed archives with all other non-primary Starch tools (all tools except `starch`, `unstarch`, `starchcat`, and `sort-bed`). Thanks to Eric Haugen for the bug report!
+ * Fixed bug with extraction of bzip2- and gzip-backed archives with all other non-primary Starch tools (all tools except :code:`starch`, :code:`unstarch`, :code:`starchcat`, and :code:`sort-bed`). Thanks to Eric Haugen for the bug report!
-------
v2.4.18
@@ -73,7 +283,7 @@ Released: **April 26, 2016**
* Starch C++ API
- * Fixed bug with extraction of bzip2- and gzip-backed archives with all other non-primary Starch tools (all tools except `starch`, `unstarch`, `starchcat`, and `sort-bed`). Thanks to Feng Tian for reports.
+ * Fixed bug with extraction of bzip2- and gzip-backed archives with all other non-primary Starch tools (all tools except :code:`starch`, :code:`unstarch`, :code:`starchcat`, and :code:`sort-bed`). Thanks to Feng Tian for reports.
-------
v2.4.16
@@ -83,11 +293,11 @@ Released: **April 5, 2016**
* :ref:`bedmap <bedmap>`
- * Added new ``--echo-ref-row-id`` option to report reference row ID elements.
+ * Added new :code:`--echo-ref-row-id` option to report reference row ID elements.
* Starch C++ API
- * Fixed bug with extraction of archives made with ``starch --gzip`` (thanks to Brad Gulko for the bug report and Paul Verhoeven and Peter Weir for compile and testing assistance).
+ * Fixed bug with extraction of archives made with :code:`starch --gzip` (thanks to Brad Gulko for the bug report and Paul Verhoeven and Peter Weir for compile and testing assistance).
* General improvements
@@ -105,7 +315,7 @@ Released: **January 21, 2016**
* :ref:`convert2bed <convert2bed>`
- * Fixed buffer overflows in ``convert2bed`` to improve conversion reliability for VCF files (thanks to Jared Andrews and Kousik Kundu for bug reports).
+ * Fixed buffer overflows in :code:`convert2bed` to improve conversion reliability for VCF files (thanks to Jared Andrews and Kousik Kundu for bug reports).
* General improvements
@@ -119,11 +329,11 @@ Released: **April 21, 2015**
* :ref:`convert2bed <convert2bed>`
- * Fixed missing ``samtools`` variable references in cluster conversion scripts (thanks to Brad Gulko for the bug report).
+ * Fixed missing :code:`samtools` variable references in cluster conversion scripts (thanks to Brad Gulko for the bug report).
* General suite-wide improvements
- * Fixed exception error message for ``stdin`` check (thanks to Brad Gulko for the bug report).
+ * Fixed exception error message for :code:`stdin` check (thanks to Brad Gulko for the bug report).
-------
@@ -134,7 +344,7 @@ Released: **April 20, 2015**
* :ref:`bedops <bedops>`
- * Resolved issue in using ``--ec`` with ``bedops`` when reading from ``stdin`` (thanks to Brad Gulko for the bug report).
+ * Resolved issue in using :code:`--ec` with :code:`bedops` when reading from :code:`stdin` (thanks to Brad Gulko for the bug report).
* General suite-wide improvements
@@ -148,22 +358,27 @@ Released: **March 13, 2015**
* :ref:`bedops <bedops>`
- * Checks have been added to determine if an integer argument is a file in the current working directory, before interpreting that argument as an overlap criterion for ``-e`` and ``-n`` options. To reduce ambiguity, if an integer is used as a file input, ``bedops`` issues a warning of the interpretation and provides guidance on how to force that value to instead be used as an overlap specification, if desired (thanks to E. Rynes for the pointer).
+ * Checks have been added to determine if an integer argument is a file in the current working directory, before interpreting that argument as an overlap criterion for :code:`-e` and :code:`-n` options.
+
+ To reduce ambiguity, if an integer is used as a file input, :code:`bedops` issues a warning of the interpretation and provides guidance on how to force that value to instead be used as an overlap specification, if desired (thanks to E. Rynes for the pointer).
* :ref:`bedmap <bedmap>`
- * Added support for ``--prec``/``--sci`` with ``--min-element`` and ``--max-element`` operations (thanks to E. Rynes for the pointer).
+ * Added support for :code:`--prec` / :code:`--sci` with :code:`--min-element` and :code:`--max-element` operations (thanks to E. Rynes for the pointer).
* :REF:`bedops <bedops>` | :ref:`bedmap <bedmap>` | :ref:`closest-features <closest-features>`
- * Added support for ``bash`` process substitution/named pipes with specification of ``--chrom`` and/or ``--ec`` options (thanks to B. Gulko for the bug report).
- * Fixed code that extracts ``gzip``-backed Starch archives from ``bedops`` and other core tools (thanks again to B. Gulko for the bug report).
+ * Added support for :code:`bash` process substitution/named pipes with specification of :code:`--chrom` and/or :code:`--ec` options (thanks to B. Gulko for the bug report).
+
+ * Fixed code that extracts :code:`gzip`-backed Starch archives from :code:`bedops` and other core tools (thanks again to B. Gulko for the bug report).
* :ref:`convert2bed <convert2bed>`
- * Switched ``matches`` and ``qSize`` fields in order of :ref:`psl2bed <psl2bed>` output. Refer to documentation for new field order.
+ * Switched :code:`matches` and :code:`qSize` fields in order of :code:`psl2bed` output. Refer to documentation for new field order.
+
* Added null sentinel to GTF ID value.
- * To help reduce the chance of buffer overflows, the `convert2bed` tool increases the maximum field length from 8191 to 24575 characters to allow parsing of inputs with longer field length, such as very long attributes from `mosquito GFF3 <https://www.vectorbase.org/download/aedes-aegypti-liverpoolbasefeaturesaaegl33gff3gz>`_ data (thanks to T. Karginov for the bug report).
+
+ * To help reduce the chance of buffer overflows, the :code:`convert2bed` tool increases the maximum field length from 8191 to 24575 characters to allow parsing of inputs with longer field length, such as very long attributes from `mosquito GFF3 <https://www.vectorbase.org/download/aedes-aegypti-liverpoolbasefeaturesaaegl33gff3gz>`_ data (thanks to T. Karginov for the bug report).
-------
v2.4.11
@@ -173,7 +388,7 @@ Released: **February 24, 2015**
* :ref:`convert2bed <convert2bed>`
- * Fixed bug in :ref:`psl2bed <psl2bed>` where ``matches`` column value was truncated by one character. Updated unit tests. Thanks to M. Wirthlin for the bug report.
+ * Fixed bug in :code:`psl2bed` where :code:`matches` column value was truncated by one character. Updated unit tests. Thanks to M. Wirthlin for the bug report.
-------
v2.4.10
@@ -183,13 +398,13 @@ Released: **February 23, 2015**
* :ref:`starch <starch>`
- * In addition to checking chromosome interleaving, the ``starch`` tool now enforces :ref:`sort-bed <sort-bed>` sort ordering on BED input and exits with an ``EINVAL`` error if the data are not sorted correctly.
+ * In addition to checking chromosome interleaving, the :code:`starch` tool now enforces :code:`sort-bed` sort ordering on BED input and exits with an :code:`EINVAL` POSIX error code if the data are not sorted correctly.
* :ref:`convert2bed <convert2bed>`
- * Added ``--zero-indexed`` option to ``wig2bed`` and ``wig2starch`` wrappers and ``convert2bed`` binary, which converts WIG data that are zero-indexed without any coordinate adjustments. This is useful for WIG data sourced from the UCSC Kent tool ``bigWigToWig``, where the ``bigWig`` data can potentially be sourced from 0-indexed BAM- or bedGraph-formatted data.
+ * Added :code:`--zero-indexed` option to :code:`wig2bed` and :code:`wig2starch` wrappers and :code:`convert2bed` binary, which converts WIG data that are zero-indexed without any coordinate adjustments. This is useful for WIG data sourced from the UCSC Kent tool :code:`bigWigToWig`, where the :code:`bigWig` data can potentially be sourced from 0-indexed BAM- or bedGraph-formatted data.
- * If the WIG input contains any element with a start coordinate of 0, the default use of ``wig2bed``, ``wig2starch`` and ``convert2bed`` will exit early with an error condition, suggesting the use of ``--zero-indexed``.
+ * If the WIG input contains any element with a start coordinate of 0, the default use of :code:`wig2bed`, :code:`wig2starch` and :code:`convert2bed` will exit early with an error condition, suggesting the use of :code:`--zero-indexed`.
* Updated copyright date range of wrapper scripts
@@ -201,15 +416,15 @@ Released: **February 17, 2015**
* :ref:`sort-bed <sort-bed>`
- * Added support for ``--check-sort`` to report if input is sorted (or not)
+ * Added support for :code:`--check-sort` to report if input is sorted (or not)
* Starch
- * Improved support for ``starch --header``, where header contains tab-delimited fields
+ * Improved support for :code:`starch --header`, where header contains tab-delimited fields
* Starch C++ API
- * Fixed bug with ``starch --header`` functionality, such that BEDOPS core tools (``bedops``, etc.) would be unable to extract correct data from headered Starch archive
+ * Fixed bug with :code:`starch --header` functionality, such that BEDOPS core tools (:code:`bedops`, etc.) would be unable to extract correct data from headered Starch archive
------
v2.4.8
@@ -235,9 +450,11 @@ Released: **February 2, 2015**
* :ref:`convert2bed <convert2bed>` fixes and improvements
- * Fixed `--split` support in :ref:`psl2bed <psl2bed>` (thanks to Marco A.)
+ * Fixed :code:`--split` support in :code:`psl2bed` (thanks to Marco A.)
+
* Fixed compilation warning regarding comparison of signed and unsigned values
- * Fixed corrupted :ref:`psl2bed <psl2bed>` test inputs
+
+ * Fixed corrupted :code:`psl2bed` test inputs
------
v2.4.6
@@ -247,19 +464,19 @@ Released: **January 30, 2015**
* :ref:`convert2bed <convert2bed>` fixes and improvements
- * Added support for conversion of the `GVF file format <http://www.sequenceontology.org/resources/gvf.html#summary>`_, including wrapper scripts and unit tests. Refer to the :ref:`gvf2bed <gvf2bed>` documentation for more information.
+ * Added support for conversion of the `GVF file format <http://www.sequenceontology.org/resources/gvf.html#summary>`_, including wrapper scripts and unit tests. Refer to the :code:`gvf2bed` documentation for more information.
- * Fixed bug in string copy of zero-length element attribute for :ref:`gff2bed` and :ref:`gtf2bed` (GFF and GTF) formats
+ * Fixed bug in string copy of zero-length element attribute for :code:`gff2bed` and :code:`gtf2bed` (GFF and GTF) formats
* General fixes and improvements
* Fixed possibly corrupt bzip2, Jansson and zlib tarballs (thanks to rekado, Shane N. and Richard S.)
- * Fixed typo in :ref:`bedextract <bedextract>` documentation
+ * Fixed typo in :code:`bedextract` documentation
* Fixed broken image in :ref:`Overview <overview>`
- * Removed 19 MB ``_build`` intermediate result directory (which should improve overall ``git clone`` time considerably!)
+ * Removed 19 MB :code:`_build` intermediate result directory (which should improve overall :code:`git clone` time considerably!)
------
v2.4.5
@@ -267,9 +484,9 @@ v2.4.5
Released: **January 28, 2015**
-* ``convert2bed`` improvements
+* :ref:`convert2bed <convert2bed>` improvements
- * Addition of RepeatMasker annotation output (``.out``) file conversion support, ``rmsk2bed`` and ``rmsk2starch`` wrappers, and unit tests
+ * Addition of RepeatMasker annotation output (:code:`.out`) file conversion support, :code:`rmsk2bed` and :code:`rmsk2starch` wrappers, and unit tests
------
v2.4.4
@@ -291,21 +508,21 @@ Released: **December 18, 2014**
* Compilation improvements
- * Shane Neph put in a great deal of work to enable parallel builds (*e.g.*, ``make -j N`` to build various targets in parallel). Depending on the end user's environment, this can speed up compilation time by a factor of 2, 4 or more.
+ * Shane Neph put in a great deal of work to enable parallel builds (*e.g.*, :code:`make -j N` to build various targets in parallel). Depending on the end user's environment, this can speed up compilation time by a factor of 2, 4 or more.
- * Fixed numerous compilation warnings of debug builds of :ref:`starch` toolkit under RHEL6/GCC and OS X 10.10.1/LLVM.
+ * Fixed numerous compilation warnings of debug builds of :code:`starch` toolkit under RHEL6/GCC and OS X 10.10.1/LLVM.
* New :ref:`bedops` features
- * Added ``--chop`` and ``--stagger`` options to "melt" inputs into contiguous or staggered disjoint regions of equivalent size.
+ * Added :code:`--chop` and :code:`--stagger` options to "melt" inputs into contiguous or staggered disjoint regions of equivalent size.
- * For less confusion, arguments for ``--element-of``, ``--chop`` and other ``bedops`` operations that take numerical modifiers no longer require a leading hyphen character. For instance, ``--element-of 1`` is now equivalent to the former usage of ``--element-of -1``.
+ * For less confusion, arguments for :code:`--element-of`, :code:`--chop` and other :code:`bedops` operations that take numerical modifiers no longer require a leading hyphen character. For instance, :code:`--element-of 1` is now equivalent to the former usage of :code:`--element-of -1`.
* New :ref:`bedmap` features
- * The ``--sweep-all`` option reads through the entire map file without early termination and can help deal with ``SIGPIPE`` errors. It adds to execution time, but the penalty is not as severe as with the use of ``--ec``. Using ``--ec`` alone will enable error checking, but will now no longer read through the entire map file. The ``--ec`` option can be used in conjunction with ``--sweep-all``, with the associated time penalties. (Another method for dealing with issue this is to override [...]
+ * The :code:`--sweep-all` option reads through the entire map file without early termination and can help deal with :code:`SIGPIPE` errors. It adds to execution time, but the penalty is not as severe as with the use of :code:`--ec`. Using :code:`--ec` alone will enable error checking, but will now no longer read through the entire map file. The :code:`--ec` option can be used in conjunction with :code:`--sweep-all`, with the associated time penalties. (Another method for dealing with i [...]
- * New ``--echo-ref-size`` and ``--echo-ref-name`` operations report genomic length of reference element, and rename the reference element in ``chrom:start-end`` (useful for labeling rows for input for ``matrix2png`` or ``R`` or other applications).
+ * New :code:`--echo-ref-size` and :code:`--echo-ref-name` operations report genomic length of reference element, and rename the reference element in :code:`chrom:start-end` (useful for labeling rows for input for :code:`matrix2png` or :code:`R` or other applications).
* :ref:`bedextract`
@@ -313,21 +530,21 @@ Released: **December 18, 2014**
* :ref:`conversion scripts <conversion_scripts>`
- * Brand new C99 binary called :ref:`convert2bed`, which wrapper scripts (``bam2bed``, etc.) now call. No more Python version dependencies, and the C-based rewrite offers massive performance improvements over old Python-based scripts.
+ * Brand new C99 binary called :code:`convert2bed`, which wrapper scripts (:code:`bam2bed`, *etc.*) now call. No more Python version dependencies, and the C-based rewrite offers massive performance improvements over old Python-based scripts.
- * Added :ref:`parallel_bam2starch` script, which parallelizes creation of :ref:`Starch <starch_specification>` archive from very large BAM files in SGE environments.
+ * Added :code:`parallel_bam2starch` script, which parallelizes creation of :ref:`Starch <starch_specification>` archive from very large BAM files in SGE environments.
* Added bug fix for missing code in :ref:`starchcluster.gnu_parallel <starchcluster>` script, where the final collation step was missing.
- * The :ref:`vcf2bed` script now accepts the ``--do-not-split`` option, which prints one BED element for all alternate alleles.
+ * The :code:`vcf2bed` script now accepts the :code:`--do-not-split` option, which prints one BED element for all alternate alleles.
* :ref:`Starch <starch_specification>` archival format and compression/extraction tools
- * Added duplicate- and :ref:`nested-element <nested_elements>` flags in v2.1 of Starch metadata, which denote if a chromosome contains one or more duplicate and/or nested elements. BED files compressed with :ref:`starch` v2.5 or greater, or Starch archives updated with :ref:`starchcat` v2.5 or greater will include these values in the archive metadata. The :ref:`unstarch` extraction tool offers ``--has-duplicate`` and ``--has-nested`` options to retrieve these flag values for a specifie [...]
+ * Added duplicate- and :ref:`nested-element <nested_elements>` flags in v2.1 of Starch metadata, which denote if a chromosome contains one or more duplicate and/or nested elements. BED files compressed with :code:`starch` v2.5 or greater, or Starch archives updated with :code:`starchcat` v2.5 or greater will include these values in the archive metadata. The :code:`unstarch` extraction tool offers :code:`--has-duplicate` and :code:`--has-nested` options to retrieve these flag values for [...]
- * Added ``--is-starch`` option to :ref:`unstarch` to test if specified input file is a Starch v1 or v2 archive.
+ * Added :code:`--is-starch` option to :code:`unstarch` to test if specified input file is a Starch v1 or v2 archive.
- * Added bug fix for compressing BED files with :ref:`starch`, where the archive would not include the last element of the BED input, if the BED input lacked a trailing newline. The compression tools now include a routine for capturing the last line, if there is no newline.
+ * Added bug fix for compressing BED files with :code:`starch`, where the archive would not include the last element of the BED input, if the BED input lacked a trailing newline. The compression tools now include a routine for capturing the last line, if there is no newline.
* Documentation improvements
@@ -341,13 +558,13 @@ Released: **April 10, 2014**
* :ref:`conversion scripts <conversion_scripts>`
- * Added support for :ref:`sort-bed` ``--tmpdir`` option to conversion scripts, to allow specification of alternative temporary directory for sorted results when used in conjunction with ``--max-mem`` option.
+ * Added support for :code:`sort-bed --tmpdir` option to conversion scripts, to allow specification of alternative temporary directory for sorted results when used in conjunction with :code:`--max-mem` option.
- * Added support for GFF3 files which include a FASTA directive in ``gff2bed`` and ``gff2starch`` (thanks to Keith Hughitt).
+ * Added support for GFF3 files which include a FASTA directive in :code:`gff2bed` and :code:`gff2starch` (thanks to Keith Hughitt).
- * Extended support for Python-based conversion scripts to support use with Python v2.6.2 and forwards, except for ``sam2bed`` and ``sam2starch``, which still require Python v2.7 or greater (and under Python3).
+ * Extended support for Python-based conversion scripts to support use with Python v2.6.2 and forwards, except for :code:`sam2bed` and :code:`sam2starch`, which still require Python v2.7 or greater (and under Python3).
- * Fixed ``--insertions`` option in :ref:`vcf2bed` to now report a single-base BED element (thanks to Matt Maurano).
+ * Fixed :code:`--insertions` option in :code:`vcf2bed` to now report a single-base BED element (thanks to Matt Maurano).
------
v2.4.1
@@ -357,33 +574,33 @@ Released: **February 26, 2014**
* :ref:`bedmap`
- * Added ``--fraction-both`` and ``--exact`` (``--fraction-both 1``) to list of compatible overlap options with ``--faster``.
+ * Added :code:`--fraction-both` and :code:`--exact` (:code:`--fraction-both 1`) to list of compatible overlap options with :code:`--faster`.
- * Added 5% performance improvement with `bedmap` operations without ``--faster``.
+ * Added 5% performance improvement with :code:`bedmap` operations without :code:`--faster`.
* Fixed scenario that can yield incorrect results (cf. `Issue 43 <https://github.com/bedops/bedops/issues/43>`_).
* :ref:`sort-bed`
- * Added ``--tmpdir`` option to allow specification of an alternative temporary directory, when used in conjunction with ``--max-mem`` option. This is useful if the host operating system's standard temporary directory (*e.g.*, ``/tmp`` on Linux or OS X) does not have sufficient space to hold intermediate results.
+ * Added :code:`--tmpdir` option to allow specification of an alternative temporary directory, when used in conjunction with :code:`--max-mem` option. This is useful if the host operating system's standard temporary directory (*e.g.*, :code:`/tmp` on Linux or OS X) does not have sufficient space to hold intermediate results.
* All :ref:`conversion scripts <conversion_scripts>`
* Improvements to error handling in Python-based conversion scripts, in the case where no input is specified.
- * Fixed typos in :ref:`gff2bed` and :ref:`psl2bed` documentation (cf. `commit a091e18 <https://github.com/bedops/bedops/commit/a091e18>`_).
+ * Fixed typos in :code:`gff2bed` and :code:`psl2bed` documentation (cf. `commit a091e18 <https://github.com/bedops/bedops/commit/a091e18>`_).
* OS X compilation improvements
* We have completed changes to the OS X build process for the remaining half of the BEDOPS binaries, which now allows direct, full compilation with Clang/LLVM (part of the Apple Xcode distribution).
- All OS X BEDOPS binaries now use Apple's system-level C++ library, instead of GNU's ``libstdc++``. It is no longer required (or recommended) to use GNU gcc to compile BEDOPS on OS X.
+ All OS X BEDOPS binaries now use Apple's system-level C++ library, instead of GNU's :code:`libstdc++`. It is no longer required (or recommended) to use GNU :code:`gcc` to compile BEDOPS on OS X.
Compilation is faster and simpler, and we can reduce the size and complexity of Mac OS X builds and installer packages. By using Apple's C++ library, we also eliminate the likelihood of missing library errors.
In the longer term, this gets us closer to moving BEDOPS to using the CMake build system, to further abstract and simplify the build process.
-* Cleaned up various compilation warnings found with ``clang``/``clang++`` and GCC kits.
+* Cleaned up various compilation warnings found with :code:`clang` / :code:`clang++` and GCC kits.
------
v2.4.0
@@ -393,9 +610,9 @@ Released: **January 9, 2014**
* :ref:`bedmap`
- * Added new ``--echo-map-size`` and ``--echo-overlap-size`` options to calculate sizes of mapped elements and overlaps between mapped and reference elements.
+ * Added new :code:`--echo-map-size` and :code:`--echo-overlap-size` options to calculate sizes of mapped elements and overlaps between mapped and reference elements.
- * Improved performance for all ``--echo-map-*`` operations.
+ * Improved performance for all :code:`--echo-map-*` operations.
* Updated documentation.
@@ -407,7 +624,7 @@ Released: **January 9, 2014**
* Added support for millions of distinct chromosomes.
- * Improved internal estimation of memory usage with ``--max-mem`` option.
+ * Improved internal estimation of memory usage with :code:`--max-mem` option.
* Added support for compilation on Cygwin (64-bit). Refer to the :ref:`installation documentation <installation_via_source_code_on_cygwin>` for build instructions.
@@ -417,39 +634,39 @@ Released: **January 9, 2014**
* All :ref:`conversion scripts <conversion_scripts>`
- * Python-based scripts no longer use temporary files, which reduces file I/O and improves performance. This change also reduces the need for large amounts of free space in a user's ``/tmp`` folder, particularly relevant for users converting multi-GB BAM files.
+ * Python-based scripts no longer use temporary files, which reduces file I/O and improves performance. This change also reduces the need for large amounts of free space in a user's :code:`/tmp` folder, particularly relevant for users converting multi-GB BAM files.
- * We now test for ability to locate ``starch``, ``sort-bed``, ``wig2bed_bin`` and ``samtools`` in user environment, quitting with the appropriate error state if the dependencies cannot be found.
+ * We now test for ability to locate :code:`starch`, :code:`sort-bed`, :code:`wig2bed_bin` and :code:`samtools` in user environment, quitting with the appropriate error state if the dependencies cannot be found.
* Improved documentation. In particular, we have added descriptive tables to each script's documentation page which describe how columns map from original data input to BED output.
* :ref:`bam2bed` and :ref:`sam2bed`
- * Added ``--custom-tags <value>`` command-line option to support a comma-separated list of custom tags (cf. `Biostars discussion <http://www.biostars.org/p/87062/>`_), *i.e.*, tags which are not part of the original SAMtools specification.
+ * Added :code:`--custom-tags <value>` command-line option to support a comma-separated list of custom tags (cf. `Biostars discussion <http://www.biostars.org/p/87062/>`_), *i.e.*, tags which are not part of the original SAMtools specification.
- * Added ``--keep-header`` option to preserve header and metadata as BED elements that use ``_header`` as the chromosome name. This now makes these conversion scripts fully "non-lossy".
+ * Added :code:`--keep-header` option to preserve header and metadata as BED elements that use :code:`_header` as the chromosome name. This now makes these conversion scripts fully "non-lossy".
* :ref:`vcf2bed`
- * Added new ``--snvs``, ``--insertions`` and ``--deletions`` options that filter VCF variants into three separate subcategories.
+ * Added new :code:`--snvs`, :code:`--insertions` and :code:`--deletions` options that filter VCF variants into three separate subcategories.
- * Added ``--keep-header`` option to preserve header and metadata as BED elements that use ``_header`` as the chromosome name. This now makes these conversion scripts fully "non-lossy".
+ * Added :code:`--keep-header` option to preserve header and metadata as BED elements that use :code:`_header` as the chromosome name. This now makes these conversion scripts fully "non-lossy".
* :ref:`gff2bed`
- * Added ``--keep-header`` option to preserve header and metadata as BED elements that use ``_header`` as the chromosome name. This now makes these conversion scripts fully "non-lossy".
+ * Added :code:`--keep-header` option to preserve header and metadata as BED elements that use :code:`_header` as the chromosome name. This now makes these conversion scripts fully "non-lossy".
* :ref:`psl2bed`
- * Added ``--keep-header`` option to preserve header and metadata as BED elements that use ``_header`` as the chromosome name. This now makes these conversion scripts fully "non-lossy".
+ * Added :code:`--keep-header` option to preserve header and metadata as BED elements that use :code:`_header` as the chromosome name. This now makes these conversion scripts fully "non-lossy".
* :ref:`wig2bed`
- * Added ``--keep-header`` option to :ref:`wig2bed` binary and ``wig2bed``/``wig2starch`` wrapper scripts, to preserve header and metadata as BED elements that use ``_header`` as the chromosome name. This now makes these conversion scripts fully "non-lossy".
+ * Added :code:`--keep-header` option to :code:`wig2bed` binary and :code:`wig2bed` / :code:`wig2starch` wrapper scripts, to preserve header and metadata as BED elements that use :code:`_header` as the chromosome name. This now makes these conversion scripts fully "non-lossy".
* Added OS X uninstaller project to allow end user to more easily remove BEDOPS tools from this platform.
-* Cleaned up various compilation warnings found with ``clang``/``clang++`` and GCC kits.
+* Cleaned up various compilation warnings found with :code:`clang` / :code:`clang++` and GCC kits.
------
v2.3.0
@@ -461,33 +678,33 @@ Released: **October 2, 2013**
* Due to changes with Google Code hosting policies at the end of the year, we have decided to change our process for distributing code, packages and documentation. While most of the work is done, we appreciate feedback on any problems you may encounter. Please email us at `bedops at stamlab.org <mailto:bedops at stamlab.org>`_ with details.
- * Migration to Github should facilitate requests for code by those who are familiar with ``git`` and want to fork our project to submit `pull requests <https://help.github.com/articles/using-pull-requests>`_.
+ * Migration to Github should facilitate requests for code by those who are familiar with :code:`git` and want to fork our project to submit `pull requests <https://help.github.com/articles/using-pull-requests>`_.
* :ref:`bedops`
- * General ``--ec`` performance improvements.
+ * General :code:`--ec` performance improvements.
* :ref:`bedmap`
- * Adds support for the new ``--skip-unmapped`` option, which filters out reference elements which do not have mapped elements associated with them. See the end of the :ref:`score operations <bedmap_score_operations>` section of the :ref:`bedmap` documentation for more detail.
+ * Adds support for the new :code:`--skip-unmapped` option, which filters out reference elements which do not have mapped elements associated with them. See the end of the :ref:`score operations <bedmap_score_operations>` section of the :ref:`bedmap` documentation for more detail.
- * General ``--ec`` performance improvements.
+ * General :code:`--ec` performance improvements.
* :ref:`starch`
- * Fixed bug with :ref:`starch` where zero-byte BED input (*i.e.*, an "empty set") created a truncated and unusable archive. We now put in a "dummy" chromosome for zero-byte input, which :ref:`unstarch` can now unpack.
+ * Fixed bug with :code:`starch` where zero-byte BED input (*i.e.*, an "empty set") created a truncated and unusable archive. We now put in a "dummy" chromosome for zero-byte input, which :code:`unstarch` can now unpack.
- This should simplify error handling with certain pipelines, specifically where set or other BEDOPS operations yield an "empty set" BED file that is subsequently compressed with :ref:`starch`.
+ This should simplify error handling with certain pipelines, specifically where set or other BEDOPS operations yield an "empty set" BED file that is subsequently compressed with :code:`starch`.
* :ref:`unstarch`
- * Can now unpack zero-byte ("empty set") compressed :ref:`starch` archive (see above).
+ * Can now unpack zero-byte ("empty set") compressed :code:`starch` archive (see above).
- * Changed ``unstarch --list`` option to print to ``stdout`` stream (this was previously sent to ``stderr``).
+ * Changed :code:`unstarch --list` option to print to :code:`stdout` stream (this was previously sent to :code:`stderr`).
* :ref:`starch` metadata library
- * Fixed array overflow bug with BEDOPS tools that take :ref:`starch <starch_specification>` archives as inputs, which affected use of archives as inputs to :ref:`closest-features`, :ref:`bedops` and :ref:`bedmap`.
+ * Fixed array overflow bug with BEDOPS tools that take :ref:`starch <starch_specification>` archives as inputs, which affected use of archives as inputs to :code:`closest-features`, :code:`bedops` and :code:`bedmap`.
* All :ref:`conversion scripts <conversion_scripts>`
@@ -495,25 +712,25 @@ Released: **October 2, 2013**
* Improved (more "Pythonic") error code handling.
- * Disabled support for ``--max-mem`` sort parameter until :ref:`sort-bed` `issue <https://github.com/bedops/bedops/issues/1>`_ is resolved. Scripts will continue to sort, but they will be limited to available system memory. If you are processing files larger than system memory, please contact us at `bedops at stamlab.org <mailto:bedops at stamlab.org>`_ for details of a temporary workaround.
+ * Disabled support for :code:`--max-mem` sort parameter until :ref:`sort-bed` `issue <https://github.com/bedops/bedops/issues/1>`_ is resolved. Scripts will continue to sort, but they will be limited to available system memory. If you are processing files larger than system memory, please contact us at `bedops at stamlab.org <mailto:bedops at stamlab.org>`_ for details of a temporary workaround.
* :ref:`gff2bed` conversion script
- * Resolved ``IndexError`` exceptions by fixing header support, bringing script in line with `v1.21 GFF3 spec <http://www.sequenceontology.org/gff3.shtml>`_.
+ * Resolved :code:`IndexError` exceptions by fixing header support, bringing script in line with `v1.21 GFF3 spec <http://www.sequenceontology.org/gff3.shtml>`_.
* :ref:`bam2bed` and :ref:`sam2bed` conversion scripts
- * Rewritten ``bam2*`` and ``sam2*`` scripts from ``bash`` into Python (v2.7+ support).
+ * Rewritten :code:`bam2*` and :code:`sam2*` scripts from :code:`bash` into Python (v2.7+ support).
* Improved BAM and SAM input validation against the `v1.4 SAM spec <http://samtools.sourceforge.net/SAMv1.pdf>`_.
- * New ``--split`` option prints reads with ``N`` CIGAR operations as separated BED elements.
+ * New :code:`--split` option prints reads with :code:`N` CIGAR operations as separated BED elements.
- * New ``--all-reads`` option prints all reads, mapped and unmapped.
+ * New :code:`--all-reads` option prints all reads, mapped and unmapped.
* :ref:`bedextract`
- * Fixed ``stdin`` bug with :ref:`bedextract`.
+ * Fixed :code:`stdin` bug with :code:`bedextract`.
* New documentation via `readthedocs.org <readthedocs.org>`_.
@@ -527,11 +744,11 @@ Released: **October 2, 2013**
* OS X compilation improvements
- * We have made changes to the OS X build process for half of the BEDOPS binaries, which allows direct compilation with Clang/LLVM (part of the Apple Xcode distribution). Those binaries now use Apple's system-level C++ library, instead of GNU's ``libstdc++``.
+ * We have made changes to the OS X build process for half of the BEDOPS binaries, which allows direct compilation with Clang/LLVM (part of the Apple Xcode distribution). Those binaries now use Apple's system-level C++ library, instead of GNU's :code:`libstdc++`.
This change means that we require Mac OS X 10.7 ("Lion") or greater |---| we do not support 10.6 at this time.
- Compilation is faster and simpler, and we can reduce the size and complexity of Mac OS X builds and installer packages. By using Apple's C++ library, we also reduce the likelihood of missing library errors. When this process is completed for the remaining binaries, it will no longer be necessary to install GCC 4.7+ (by way of MacPorts or other package managers) in order to build BEDOPS on OS X, nor will we have to bundle ``libstdc++`` with the installer.
+ Compilation is faster and simpler, and we can reduce the size and complexity of Mac OS X builds and installer packages. By using Apple's C++ library, we also reduce the likelihood of missing library errors. When this process is completed for the remaining binaries, it will no longer be necessary to install GCC 4.7+ (by way of MacPorts or other package managers) in order to build BEDOPS on OS X, nor will we have to bundle :code:`libstdc++` with the installer.
-------
v2.2.0b
@@ -551,11 +768,11 @@ Released: **May 22, 2013**
* :ref:`Starch v2 test suite <starch_specification>`
- * We have added a test suite for the Starch archive toolkit with the source download. Test inputs include randomized BED data generated from chromosome and bounds data stored on UCSC servers as well as static FIMO search results. Tests put :ref:`starch`, :ref:`unstarch` and :ref:`starchcat` through various usage scenarios. Please refer to the Starch-specific Makefiles and the test target and subfolder's `README` doc for more information.
+ * We have added a test suite for the Starch archive toolkit with the source download. Test inputs include randomized BED data generated from chromosome and bounds data stored on UCSC servers as well as static FIMO search results. Tests put :code:`starch`, :code:`unstarch` and :code:`starchcat` through various usage scenarios. Please refer to the Starch-specific Makefiles and the test target and subfolder's `README` doc for more information.
* :ref:`starchcat`
- * Resolves bug with ``--gzip`` option, allowing updates of ``gzip`` -backed v1.2 and v1.5 archives to the :ref:`v2 Starch format <starch_specification>` (either ``bzip2`` - or ``gzip`` -backed).
+ * Resolves bug with :code:`--gzip` option, allowing updates of :code:`gzip` -backed v1.2 and v1.5 archives to the :ref:`v2 Starch format <starch_specification>` (either :code:`bzip2` - or :code:`gzip` -backed).
* :ref:`unstarch`
@@ -563,7 +780,7 @@ Released: **May 22, 2013**
* :ref:`conversion scripts <conversion_scripts>`
- * We have partially reverted :ref:`wig2bed`, providing a Bash shell wrapper to the original C binary. This preserves consistency of command-line options across the conversion suite, while making use of the C binary to recover performance lost from the Python-based v2.1 revision of :ref:`wig2bed` (which at this time is no longer supported). (Thanks to Matt Maurano for reporting this issue.)
+ * We have partially reverted :code:`wig2bed`, providing a Bash shell wrapper to the original C binary. This preserves consistency of command-line options across the conversion suite, while making use of the C binary to recover performance lost from the Python-based v2.1 revision of :code:`wig2bed` (which at this time is no longer supported). (Thanks to Matt Maurano for reporting this issue.)
------
v2.1.1
@@ -573,15 +790,15 @@ Released: **May 3, 2013**
* :ref:`bedmap`
- * Major performance improvements made in v2.1.1, such that current :ref:`bedmap` now operates as fast or faster than the v1.2.5 version of :ref:`bedmap`!
+ * Major performance improvements made in v2.1.1, such that current :code:`bedmap` now operates as fast or faster than the v1.2.5 version of :code:`bedmap`!
* :ref:`bedops`
- * Resolves bug with ``--partition`` option.
+ * Resolves bug with :code:`--partition` option.
* :ref:`conversion scripts <conversion_scripts>`
- * All v2.1.0 Python-based scripts now include fix for ``SIGPIPE`` handling, such that use of ``head`` or other common UNIX utilities to process buffered standard output no longer yields ``IOError`` exceptions. (Thanks to Matt Maurano for reporting this bug.)
+ * All v2.1.0 Python-based scripts now include fix for :code:`SIGPIPE` handling, such that use of :code:`head` or other common UNIX utilities to process buffered standard output no longer yields :code:`IOError` exceptions. (Thanks to Matt Maurano for reporting this bug.)
* 32-bit Linux binary support
@@ -601,25 +818,25 @@ Released: **April 22, 2013**
* :ref:`bedops`
- * New ``--partition`` operator efficiently generates disjoint segments made from genomic boundaries of all overlapping inputs.
+ * New :code:`--partition` operator efficiently generates disjoint segments made from genomic boundaries of all overlapping inputs.
* :ref:`conversion scripts <conversion_scripts>`
- * All scripts now use :ref:`sort-bed` behind the scenes to output sorted BED output, ready for use with BEDOPS utilities. It is no longer necessary to pipe data to or otherwise post-process converted data with :ref:`sort-bed`.
+ * All scripts now use :code:`sort-bed` behind the scenes to output sorted BED output, ready for use with BEDOPS utilities. It is no longer necessary to pipe data to or otherwise post-process converted data with :code:`sort-bed`.
- * New :ref:`psl2bed` conversion script, converting `PSL-formatted UCSC BLAT output <http://genome.ucsc.edu/FAQ/FAQformat.html#format2>`_ to BED.
+ * New :code:`psl2bed` conversion script, converting `PSL-formatted UCSC BLAT output <http://genome.ucsc.edu/FAQ/FAQformat.html#format2>`_ to BED.
- * New :ref:`wig2bed` conversion script written in Python.
+ * New :code:`wig2bed` conversion script written in Python.
- * New ``*2starch`` :ref:`conversion scripts <conversion_scripts>` offered for all ``*2bed`` scripts, which output Starch v2 archives.
+ * New :code:`*2starch` :ref:`conversion scripts <conversion_scripts>` offered for all :code:`*2bed` scripts, which output Starch v2 archives.
* :ref:`closest-features`
- * Replaced ``--shortest`` option name with ``--closest``, for clarity. (Old scripts which use ``--shortest`` will continue to work with the deprecated option name for now. We advise editing pipelines, as needed.)
+ * Replaced :code:`--shortest` option name with :code:`--closest`, for clarity. (Old scripts which use :code:`--shortest` will continue to work with the deprecated option name for now. We advise editing pipelines, as needed.)
* :ref:`starch`
- * Improved error checking for interleaved records. This also makes use of ``*2starch`` conversion scripts with the ``--do-not-sort`` option safer.
+ * Improved error checking for interleaved records. This also makes use of :code:`*2starch` conversion scripts with the :code:`--do-not-sort` option safer.
* Improved Mac OS X support
@@ -633,9 +850,9 @@ v2.0.0b
Released: **February 19, 2013**
-* Added :ref:`starchcluster` script variant which supports task distribution with `GNU Parallel <http://www.gnu.org/software/parallel/>`_.
+* Added :code:`starchcluster` script variant which supports task distribution with `GNU Parallel <http://www.gnu.org/software/parallel/>`_.
-* Fixed minor problem with :ref:`bam2bed` and :ref:`sam2bed` conversion scripts.
+* Fixed minor problem with :code:`bam2bed` and :code:`sam2bed` conversion scripts.
-------
v2.0.0a
@@ -647,41 +864,41 @@ Released: **February 7, 2013**
* Takes in Starch-formatted archives as input, as well as raw BED (i.e., it is no longer required to extract a Starch archive to an intermediate, temporary file or named pipe before applying operations).
- * New ``--chrom`` operator jumps to and operates on information for specified chromosome only.
+ * New :code:`--chrom` operator jumps to and operates on information for specified chromosome only.
- * New ``--echo-map-id-uniq`` operator lists unique IDs from overlapping mapping elements.
+ * New :code:`--echo-map-id-uniq` operator lists unique IDs from overlapping mapping elements.
- * New ``--max-element`` and ``--min-element`` operators return the highest or lowest scoring overlapping map element.
+ * New :code:`--max-element` and :code:`--min-element` operators return the highest or lowest scoring overlapping map element.
* :ref:`bedops`
* Takes in Starch-formatted archives as input, as well as raw BED.
- * New ``--chrom`` operator jumps to and operates on information for specified chromosome only.
+ * New :code:`--chrom` operator jumps to and operates on information for specified chromosome only.
* :ref:`closest-features`
* Takes in Starch-formatted archives as input, as well as raw BED.
- * New ``--chrom`` operator jumps to and operates on information for specified chromosome only.
+ * New :code:`--chrom` operator jumps to and operates on information for specified chromosome only.
* :ref:`sort-bed` and ``bbms``
- * New ``--max-mem`` option to limit system memory on large BED inputs.
+ * New :code:`--max-mem` option to limit system memory on large BED inputs.
- * Incorporated ``bbms`` functionality into :ref:`sort-bed` with use of ``--max-mem`` operator.
+ * Incorporated :code:`bbms` functionality into :code:`sort-bed` with use of :code:`--max-mem` operator.
* :ref:`starch`, :ref:`starchcat` and :ref:`unstarch`
- * New metadata enhancements to Starch-format archival and extraction, including: ``--note``, ``--elements``, ``--bases``, ``--bases-uniq``, ``--list-chromosomes``, ``--archive-timestamp``, ``--archive-type`` and ``--archive-version`` (see ``--help`` to :ref:`starch`, :ref:`starchcat` and :ref:`unstarch` binaries, or view the documentation for these applications for more detail).
+ * New metadata enhancements to Starch-format archival and extraction, including: :code:`--note`, :code:`--elements`, :code:`--bases`, :code:`--bases-uniq`, :code:`--list-chromosomes`, :code:`--archive-timestamp`, :code:`--archive-type` and :code:`--archive-version` (see :code:`--help` to :code:`starch`, :code:`starchcat` and :code:`unstarch` binaries, or view the documentation for these applications for more detail).
- * Adds 20-35% performance boost to creating Starch archives with :ref:`starch` utility.
+ * Adds 20-35% performance boost to creating Starch archives with :code:`starch` utility.
* New documentation with technical overview of the Starch format specification.
* :ref:`conversion scripts <conversion_scripts>`
- * New :ref:`gtf2bed` conversion script, converting GTF (v2.2) to BED.
+ * New :code:`gtf2bed` conversion script, converting GTF (v2.2) to BED.
* Scripts are now part of main download; it is no longer necessary to download the BEDOPS companion separately.
@@ -693,7 +910,7 @@ Released: **January 14, 2013**
* Adds support for Apple 32- and 64-bit Intel hardware running OS X 10.5 through 10.8.
-* Adds ``README`` for companion download.
+* Adds :code:`README` for companion download.
* Removes some obsolete code.
@@ -703,9 +920,9 @@ v1.2.5
Released: **October 13, 2012**
-* Fixed unusual bug with :ref:`unstarch`, where an extra (and incorrect) line of BED data can potentially be extracted from an archive.
+* Fixed unusual bug with :code:`unstarch`, where an extra (and incorrect) line of BED data can potentially be extracted from an archive.
-* Updated companion download with updated :ref:`bam2bed` and :ref:`sam2bed` conversion scripts to address 0-indexing error with previous revisions.
+* Updated companion download with updated :code:`bam2bed` and :code:`sam2bed` conversion scripts to address 0-indexing error with previous revisions.
------
v1.2.3
@@ -713,10 +930,12 @@ v1.2.3
Released: **August 17, 2012**
-* Added ``--indicator`` option to :ref:`bedmap`.
+* Added :code:`--indicator` option to :code:`bedmap`.
* Assorted changes to conversion scripts and associated companion download.
.. |--| unicode:: U+2013 .. en dash
.. |---| unicode:: U+2014 .. em dash, trimming surrounding whitespace
:trim:
+.. role:: bash(code)
+ :language: bash
diff --git a/docs/content/summary.rst b/docs/content/summary.rst
index c26d5a2..53ec706 100644
--- a/docs/content/summary.rst
+++ b/docs/content/summary.rst
@@ -114,8 +114,13 @@ Set operation and statistical utilities
+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
| ``--max`` | Reports the highest score from overlapping elements in ``map-file``. | 1 | 2 | 5 |
+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
-| ``--max-element`` | An element with the highest score from overlapping elements in | 1 | 2 | 5 |
-| | ``map-file``. If no overlapping element exists, ``NAN`` is reported. | | | |
+| ``--max-element`` | The lexicographically "smallest" element with the highest score from | 1 | 2 | 5 |
+| | overlapping elements in ``map-file``. If no overlapping element | | | |
+| | exists, ``NAN`` is reported (unless ``--skip-unmapped`` is used). | | | |
++-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--max-element-rand`` | A randomly-chosed element with the highest score from overlapping | 1 | 2 | 5 |
+| | elements in ``map-file``. If no overlapping element exists, ``NAN`` | | | |
+| | is reported (unless ``--skip-unmapped`` is used). | | | |
+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
| ``--mean`` | Reports the average score from overlapping elements in ``map-file``. | 1 | 2 | 5 |
+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
@@ -123,8 +128,13 @@ Set operation and statistical utilities
+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
| ``--min`` | Reports the lowest score from overlapping elements in ``map-file``. | 1 | 2 | 5 |
+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
-| ``--min-element`` | An element with the lowest score from overlapping elements in | 1 | 2 | 5 |
-| | ``map-file``. If no overlapping element exists, ``NAN`` is reported. | | | |
+| ``--min-element`` | The lexicographically "smallest" element with the lowest score from | 1 | 2 | 5 |
+| | overlapping elements in ``map-file``. If no overlapping element | | | |
+| | exists, ``NAN`` is reported (unless ``--skip-unmapped`` is used). | | | |
++-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--min-element-rand`` | A randomly-chosed element with the lowest score from overlapping | 1 | 2 | 5 |
+| | elements in ``map-file``. If no overlapping element exists, ``NAN`` | | | |
+| | is reported (unless ``--skip-unmapped`` is used). | | | |
+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
| ``--skip-unmapped`` | Omits printing reference elements which do not associate with any | 1 | 2 | 3 |
| | mapped elements. | | | |
@@ -272,6 +282,8 @@ Compression and extraction
+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
| ``--note="foo bar..."`` | Append note to output archive metadata (optional). | 1 | 1 | 3 |
+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--report-progress=N`` | Write progress to standard error stream for every N input elements. | 1 | 1 | 3 |
++-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
------------
``unstarch``
@@ -280,49 +292,57 @@ Compression and extraction
* Extraction of a ``starch`` archive or attributes.
* BEDOPS :ref:`unstarch` documentation.
-+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
-| option | description | min. file inputs | max. file inputs | min. BED columns |
-+===============================+======================================================================+==================+==================+==================+
-| (no option) | NA | 1 | 1 | NA |
-+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
-| ``--archive-type`` | Show archive's compression type (either ``bzip2`` or ``gzip``). | 1 | 1 | NA |
-+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
-| ``--archive-version`` | Show archive version (at this time, either 1.x or 2.x). | 1 | 1 | NA |
-+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
-| ``--archive-timestamp`` | Show archive creation timestamp (ISO 8601 format). | 1 | 1 | NA |
-+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
-| ``--bases <chromosome>`` | Show total, non-unique base counts for optional ``<chromosome>`` | 1 | 1 | NA |
-| | (omitting ``<chromosome>`` shows total non-unique base count). | | | |
-+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
-| ``--bases-uniq <chromosome>`` | Show unique base counts for optional ``<chromosome>`` (omitting | 1 | 1 | NA |
-| | ``<chromosome>`` shows total, unique base count). | | | |
-+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
-| ``<chromosome>`` | Decompress information for a single ``<chromosome>`` only. | 1 | 1 | NA |
-+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
-| ``--duplicatesExist`` or | Report if optional ``<chromosome>`` or chromosomes contain duplicate | 1 | 1 | NA |
-| ``--duplicatesExistAsString`` | elements as 0/1 numbers or false/true strings | | | |
-| with ``<chromosome>`` | | | | |
-+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
-| ``--elements <chromosome>`` | Show element count for optional ``<chromosome>`` (omitting | 1 | 1 | NA |
-| | ``<chromosome>`` shows total element count). | | | |
-+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
-| ``--is-starch`` | Test if the <starch-file> is a valid starch archive, returning 0/1 | 1 | 1 | NA |
-| | for a false/true result | | | |
-+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
-| ``--list`` or ``--list-json`` | Print the metadata for a ``starch`` file, either in tabular form or | 1 | 1 | NA |
-| | with JSON formatting. | | | |
-+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
-| ``--list-chr`` or | List all chromosomes in ``starch`` archive (similar to | 1 | 1 | NA |
-| ``--list-chromosomes`` | ``bedextract --list-chr``). | | | |
-+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
-| ``--nestedsExist`` or | Report if optional ``<chromosome>`` or chromosomes contain nested | 1 | 1 | NA |
-| ``--nestedsExistAsString`` | elements as 0/1 numbers or false/true strings | | | |
-| with ``<chromosome>`` | | | | |
-+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
-| ``--note`` | Show descriptive note (if originally added to archive). | 1 | 1 | NA |
-+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
-| ``--sha1-signature`` | Show SHA1 signature of JSON-formatted metadata (Base64-encoded). | 1 | 1 | NA |
-+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
++------------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| option | description | min. file inputs | max. file inputs | min. BED columns |
++====================================+======================================================================+==================+==================+==================+
+| (no option) | NA | 1 | 1 | NA |
++------------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--archive-type`` | Show archive's compression type (either ``bzip2`` or ``gzip``). | 1 | 1 | NA |
++------------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--archive-version`` | Show archive version (at this time, either 1.x or 2.x). | 1 | 1 | NA |
++------------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--archive-timestamp`` | Show archive creation timestamp (ISO 8601 format). | 1 | 1 | NA |
++------------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--bases <chromosome>`` | Show total, non-unique base counts for optional ``<chromosome>`` | 1 | 1 | NA |
+| | (omitting ``<chromosome>`` shows total non-unique base count). | | | |
++------------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--bases-uniq <chromosome>`` | Show unique base counts for optional ``<chromosome>`` (omitting | 1 | 1 | NA |
+| | ``<chromosome>`` shows total, unique base count). | | | |
++------------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``<chromosome>`` | Decompress information for a single ``<chromosome>`` only. | 1 | 1 | NA |
++------------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--duplicatesExist`` or | Report if optional ``<chromosome>`` or chromosomes contain duplicate | 1 | 1 | NA |
+| ``--duplicatesExistAsString`` | elements as 0/1 numbers or false/true strings | | | |
+| with ``<chromosome>`` | | | | |
++------------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--elements <chromosome>`` | Show element count for optional ``<chromosome>`` (omitting | 1 | 1 | NA |
+| | ``<chromosome>`` shows total element count). | | | |
++------------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--elements-max-string-length`` | Show element maximum string length for optional ``<chromosome>`` | 1 | 1 | NA |
+| | (omitting ``<chromosome>`` shows maximum string length over all | | | |
+| | chromosomes). | | | |
++------------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--is-starch`` | Test if the <starch-file> is a valid starch archive, returning 0/1 | 1 | 1 | NA |
+| | for a false/true result | | | |
++------------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--list`` or ``--list-json`` | Print the metadata for a ``starch`` file, either in tabular form or | 1 | 1 | NA |
+| | with JSON formatting. | | | |
++------------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--list-chr`` or | List all chromosomes in ``starch`` archive (similar to | 1 | 1 | NA |
+| ``--list-chromosomes`` | ``bedextract --list-chr``). | | | |
++------------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--nestedsExist`` or | Report if optional ``<chromosome>`` or chromosomes contain nested | 1 | 1 | NA |
+| ``--nestedsExistAsString`` | elements as 0/1 numbers or false/true strings | | | |
+| with ``<chromosome>`` | | | | |
++------------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--note`` | Show descriptive note (if originally added to archive). | 1 | 1 | NA |
++------------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--signature`` with | Show SHA-1 signature of specified chromosome (Base64-encoded) | 1 | 1 | NA |
+| ``<chromosome>`` | or all signatures if chromosome is not specified. | | | |
++------------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--verify-signature`` with | Compare SHA-1 signature of specified chromosome with signature that | 1 | 1 | NA |
+| ``<chromosome>`` | is stored in the archive metadata, reporting error is mismatched. | | | |
++------------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
-------------
``starchcat``
@@ -342,6 +362,25 @@ Compression and extraction
+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
| ``--note="foo bar..."`` | Append note to output archive metadata (optional). | 1 | No imposed limit | NA |
+-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--report-progress=N`` | Write progress to standard error stream for every N input elements. | 1 | No imposed limit | NA |
++-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+
+---------------
+``starchstrip``
+---------------
+
+* Extract or filter a ``starch`` archive by one or more specified chromosome names.
+* BEDOPS :ref:`starchstrip <starchstrip>` documentation.
+
++-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| option | description | min. file inputs | max. file inputs | min. BED columns |
++===============================+======================================================================+==================+==================+==================+
+| (no option) | NA | 1 | No imposed limit | NA |
++-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+| ``--include`` or ``--exclude``| Writes output with inclusion or exclusion of specified chromosome | NA | No imposed limit | NA |
+| with <chromosomes> | name records (comma-delimited string). | | | |
++-------------------------------+----------------------------------------------------------------------+------------------+------------------+------------------+
+
.. |--| unicode:: U+2013 .. en dash
.. |---| unicode:: U+2014 .. em dash, trimming surrounding whitespace
diff --git a/docs/content/usage-examples/starchcluster.rst b/docs/content/usage-examples/starchcluster.rst
index 4061b05..1fc5b50 100644
--- a/docs/content/usage-examples/starchcluster.rst
+++ b/docs/content/usage-examples/starchcluster.rst
@@ -15,15 +15,15 @@ For this script, we use :ref:`bedextract` to quickly build a list of chromosomes
Script
======
-Two versions of the ``starchcluster`` script are included with the source and package distributions of BEDOPS (see :ref:`Installation <installation>` for more detail).
+Three versions of the ``starchcluster`` script are included with the source and package distributions of BEDOPS (see :ref:`Installation <installation>` for more detail).
-One version makes use of an `Oracle Grid Engine <http://en.wikipedia.org/wiki/Oracle_Grid_Engine>`_ (or Sun Grid Engine) cluster environment to distribute per-chromosome tasks, while the other script uses `GNU Parallel <http://www.gnu.org/software/parallel/>`_ to split the workload over cores or processors on the local host.
+One version makes use of an `Oracle Grid Engine <http://en.wikipedia.org/wiki/Oracle_Grid_Engine>`_ (or Sun Grid Engine) cluster environment to distribute per-chromosome tasks, another script uses `GNU Parallel <http://www.gnu.org/software/parallel/>`_ to split the workload over cores or processors on the local host. Finally, we include a version that implements a `SLURM <https://en.wikipedia.org/wiki/Slurm_Workload_Manager>`_ -capable script.
==========
Discussion
==========
-The overview that follows applies to the Grid Engine-based version of the `starchcluster` script. However, the general algorithm is identical for both the Grid Engine- and GNU Parallel-based compression scripts.
+The overview that follows applies to the Grid Engine-based version of the `starchcluster` script. However, the general algorithm is identical for the Grid Engine-, GNU Parallel-, and SLURM-based compression scripts.
-------------------
Splitting BED files
diff --git a/docs/index.rst b/docs/index.rst
index c98ada1..151b8a8 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -57,23 +57,23 @@ Contents
<div class="col_element">
<img src="_images/linux_v2.png" style="height:26px; width: auto !important; margin-bottom:10px;">
<ul style="list-style-type:square; font-size:smaller; margin-left:0; margin-right:0px; padding-right:0px; padding-left:20px;">
- <li><a href="https://github.com/bedops/bedops/releases/download/v2.4.20/bedops_linux_x86_64-v2.4.20.tar.bz2">x86-64 (64-bit)</a> binaries</li>
- <li><a href="https://github.com/bedops/bedops/releases/download/v2.4.20/bedops_linux_i386-v2.4.20.tar.bz2">i386 (32-bit)</a> binaries</li>
+ <li><a href="https://github.com/bedops/bedops/releases/download/v2.4.26/bedops_linux_x86_64-v2.4.26.tar.bz2">x86-64 (64-bit)</a> binaries</li>
+ <li><a href="https://github.com/bedops/bedops/releases/download/v2.4.26/bedops_linux_i386-v2.4.26.tar.bz2">i386 (32-bit)</a> binaries</li>
<li><a href="content/installation.html#linux">Installation instructions</a> for Linux hosts</li>
</ul>
</div>
<div class="col_element">
<img src="_images/macosx_v2.png" style="height:26px; width: auto !important; margin-bottom:10px;">
<ul style="list-style-type:square; font-size:smaller; margin-left:0; margin-right:0px; padding-right:0px; padding-left:20px;">
- <li><a href="https://github.com/bedops/bedops/releases/download/v2.4.20/BEDOPS.2.4.20.pkg.zip">Intel (32-/64-bit, 10.7-10.11)</a> installer package</li>
+ <li><a href="https://github.com/bedops/bedops/releases/download/v2.4.26/BEDOPS.2.4.26.pkg.zip">Intel (32-/64-bit, 10.7-10.12)</a> installer package</li>
<li><a href="content/installation.html#mac-os-x">Installation instructions</a> for Mac OS X hosts</li>
</ul>
</div>
<div class="col_element">
<img src="_images/source_v2.png" style="height:26px; width: auto !important; margin-bottom:10px;">
<ul style="list-style-type:square; font-size:smaller; margin-left:0; margin-right:0px; padding-right:0px; padding-left:20px;">
- <li><a href="https://github.com/bedops/bedops/archive/v2.4.20.tar.gz">Source code</a> (tar.gz)</li>
- <li><a href="https://github.com/bedops/bedops/archive/v2.4.20.zip">Source code</a> (zip)</li>
+ <li><a href="https://github.com/bedops/bedops/archive/v2.4.26.tar.gz">Source code</a> (tar.gz)</li>
+ <li><a href="https://github.com/bedops/bedops/archive/v2.4.26.zip">Source code</a> (zip)</li>
<li><a href="content/installation.html#installation-via-source-code">Compilation instructions</a></li>
</ul>
</div>
@@ -96,8 +96,8 @@ Contents
.. |linux_downloads| raw:: html
<ul style="list-style-type:square; font-size:smaller; margin:10px; padding:0;">
- <li><a href="https://github.com/bedops/bedops/releases/download/v2.4.20/bedops_linux_x86_64-v2.4.20.tar.bz2">x86-64 (64-bit)</a> binaries</li>
- <li><a href="https://github.com/bedops/bedops/releases/download/v2.4.20/bedops_linux_i386-v2.4.20.tar.bz2">i386 (32-bit)</a> binaries</li>
+ <li><a href="https://github.com/bedops/bedops/releases/download/v2.4.26/bedops_linux_x86_64-v2.4.26.tar.bz2">x86-64 (64-bit)</a> binaries</li>
+ <li><a href="https://github.com/bedops/bedops/releases/download/v2.4.26/bedops_linux_i386-v2.4.26.tar.bz2">i386 (32-bit)</a> binaries</li>
<li><a href="content/installation.html#linux">Installation instructions</a> for Linux hosts</li>
</ul>
@@ -111,7 +111,7 @@ Contents
.. |macosx_downloads| raw:: html
<ul style="list-style-type:square; font-size:smaller; margin:10px; padding:0;">
- <li><a href="https://github.com/bedops/bedops/releases/download/v2.4.20/BEDOPS.2.4.20.pkg.zip">Intel (32-/64-bit, 10.7-10.10)</a> installer package</li>
+ <li><a href="https://github.com/bedops/bedops/releases/download/v2.4.26/BEDOPS.2.4.26.pkg.zip">Intel (32-/64-bit, 10.7-10.10)</a> installer package</li>
<li><a href="content/installation.html#mac-os-x">Installation instructions</a> for Mac OS X hosts</li>
</ul>
@@ -125,8 +125,8 @@ Contents
.. |source_downloads| raw:: html
<ul style="list-style-type:square; font-size:smaller; margin:10px; padding:0;">
- <li><a href="https://github.com/bedops/bedops/archive/v2.4.20.tar.gz">Source code</a> (tar.gz)</li>
- <li><a href="https://github.com/bedops/bedops/archive/v2.4.20.zip">Source code</a> (zip)</li>
+ <li><a href="https://github.com/bedops/bedops/archive/v2.4.26.tar.gz">Source code</a> (tar.gz)</li>
+ <li><a href="https://github.com/bedops/bedops/archive/v2.4.26.zip">Source code</a> (zip)</li>
<li><a href="content/installation.html#installation-via-source-code">Compilation instructions</a></li>
</ul>
@@ -154,6 +154,7 @@ Contents
<li><a href="content/reference/file-management/sorting/sort-bed.html"><tt>sort-bed</tt></a> - apply lexicographical sort to BED data</li>
<li><a href="content/reference/file-management/compression/starch.html"><tt>starch</tt></a> and <a href="content/reference/file-management/compression/unstarch.html"><tt>unstarch</tt></a> - compress and extract BED data</li>
<li><a href="content/reference/file-management/compression/starchcat.html"><tt>starchcat</tt></a> - merge compressed archives</li>
+ <li><a href="content/reference/file-management/compression/starchstrip.html"><tt>starchstrip</tt></a> - filter archive by chromosomes</li>
<li><a href="content/reference/file-management/conversion.html">Conversion tools</a> - convert common genomic formats to BED</li>
</ul>
</div>
@@ -172,7 +173,7 @@ Contents
<img src="_images/toc_v2.png" style="height:130px; margin:0; width:auto !important; margin-bottom:12px;">
<ul style="list-style-type:square; font-size:smaller; margin:0; margin-right:2px; padding-right:0px; padding-left:20px;">
<li><a href="content/summary.html">Table summary</a> of <strong>BEDOPS</strong> toolkit</li>
- <li><a href="content/reference/file-management/compression/starch-specification.html">Starch v2.1</a> format specification</li>
+ <li><a href="content/reference/file-management/compression/starch-specification.html">Starch v2.2</a> format specification</li>
<li><a href="content/reference/set-operations/nested-elements.html">About nested elements</a>
<li><a href="content/revision-history.html">Revision history</a></li>
<li><a href="content/release.html">Github release instructions</a></li>
@@ -238,6 +239,7 @@ Contents
<li><a href="content/reference/file-management/sorting/sort-bed.html"><tt>sort-bed</tt></a> - apply lexicographical sort to BED data</li>
<li><a href="content/reference/file-management/compression/starch.html"><tt>starch</tt></a> and <a href="content/reference/file-management/compression/unstarch.html"><tt>unstarch</tt></a> - compress and extract BED data</li>
<li><a href="content/reference/file-management/compression/starchcat.html"><tt>starchcat</tt></a> - merge compressed archives</li>
+ <li><a href="content/reference/file-management/compression/starchstrip.html"><tt>starchstrip</tt></a> - filter archives by chromosome names</li>
<li><a href="content/reference/file-management/conversion.html">Conversion tools</a> - convert common genomic formats to BED</li>
</ul>
@@ -281,7 +283,7 @@ Contents
<ul style="list-style-type:square; font-size:smaller; margin:10px; padding:0;">
<li><a href="content/summary.html">Table summary</a> of <strong>BEDOPS</strong> toolkit</li>
- <li><a href="content/reference/file-management/compression/starch-specification.html">Starch v2.1</a> format specification</li>
+ <li><a href="content/reference/file-management/compression/starch-specification.html">Starch v2.2</a> format specification</li>
<li><a href="content/reference/set-operations/nested-elements.html">About nested elements</a>
<li><a href="content/revision-history.html">Revision history</a></li>
<li><a href="content/release.html">Github release instructions</a></li>
diff --git a/interfaces/general-headers/algorithm/WindowSweep.hpp b/interfaces/general-headers/algorithm/WindowSweep.hpp
index e85a251..ba5eb32 100644
--- a/interfaces/general-headers/algorithm/WindowSweep.hpp
+++ b/interfaces/general-headers/algorithm/WindowSweep.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -71,8 +71,8 @@ namespace WindowSweep {
sweep() Assumptions:
1) in terms of RangeComp(a, b):
return 0 if a and b are 'within range' of each other
- return less than 0 if a is 'greater than' b
- return greater than 0 if a is 'less than' b
+ return less than 0 if a is 'less than' b
+ return greater than 0 if a is 'greater than' b
2) in terms of RangeComp(c, c):
return 0 : object must always be in range of itself
diff --git a/interfaces/general-headers/algorithm/bed/FindBedRange.hpp b/interfaces/general-headers/algorithm/bed/FindBedRange.hpp
index 1f0733a..3b9ea5c 100644
--- a/interfaces/general-headers/algorithm/bed/FindBedRange.hpp
+++ b/interfaces/general-headers/algorithm/bed/FindBedRange.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/BedVisitors.hpp b/interfaces/general-headers/algorithm/visitors/BedVisitors.hpp
index 936e7ad..67ee615 100644
--- a/interfaces/general-headers/algorithm/visitors/BedVisitors.hpp
+++ b/interfaces/general-headers/algorithm/visitors/BedVisitors.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -30,6 +30,7 @@
#include "bed/OvrAggregateVisitor.hpp"
#include "bed/OvrUniqueVisitor.hpp"
#include "bed/OvrUniqueFractionVisitor.hpp"
+#include "bed/WeightedAverageVisitor.hpp"
#include "NumericalVisitors.hpp"
#include "other/EchoVisitor.hpp"
#include "other/MultiVisitor.hpp"
diff --git a/interfaces/general-headers/algorithm/visitors/NumericalVisitors.hpp b/interfaces/general-headers/algorithm/visitors/NumericalVisitors.hpp
index f72f14f..fc0fbe5 100644
--- a/interfaces/general-headers/algorithm/visitors/NumericalVisitors.hpp
+++ b/interfaces/general-headers/algorithm/visitors/NumericalVisitors.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/OtherVisitors.hpp b/interfaces/general-headers/algorithm/visitors/OtherVisitors.hpp
index 4816665..f948d17 100644
--- a/interfaces/general-headers/algorithm/visitors/OtherVisitors.hpp
+++ b/interfaces/general-headers/algorithm/visitors/OtherVisitors.hpp
@@ -5,7 +5,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/VisitorFactory.hpp b/interfaces/general-headers/algorithm/visitors/VisitorFactory.hpp
index 994b007..8c317f7 100644
--- a/interfaces/general-headers/algorithm/visitors/VisitorFactory.hpp
+++ b/interfaces/general-headers/algorithm/visitors/VisitorFactory.hpp
@@ -5,7 +5,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/Visitors.hpp b/interfaces/general-headers/algorithm/visitors/Visitors.hpp
index 2af6a5d..ab43927 100644
--- a/interfaces/general-headers/algorithm/visitors/Visitors.hpp
+++ b/interfaces/general-headers/algorithm/visitors/Visitors.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/bed/BedBaseVisitor.hpp b/interfaces/general-headers/algorithm/visitors/bed/BedBaseVisitor.hpp
index 9926b6c..23b62b5 100644
--- a/interfaces/general-headers/algorithm/visitors/bed/BedBaseVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/bed/BedBaseVisitor.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -123,7 +123,7 @@ namespace Visitors {
protected:
// typedefs
- typedef std::set<MapType*, Bed::CoordAddressCompare<MapType>> OrderLesser;
+ typedef std::set<MapType*, Bed::CoordRestAddressCompare<MapType>> OrderLesser;
typedef OrderLesser OrderCache;
typedef OrderLesser OrderWin;
diff --git a/interfaces/general-headers/algorithm/visitors/bed/EchoMapBedVisitor.hpp b/interfaces/general-headers/algorithm/visitors/bed/EchoMapBedVisitor.hpp
index 53fbd44..a68849e 100644
--- a/interfaces/general-headers/algorithm/visitors/bed/EchoMapBedVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/bed/EchoMapBedVisitor.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/bed/EchoMapIntersectLengthVisitor.hpp b/interfaces/general-headers/algorithm/visitors/bed/EchoMapIntersectLengthVisitor.hpp
index 8cf33f7..68e1174 100644
--- a/interfaces/general-headers/algorithm/visitors/bed/EchoMapIntersectLengthVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/bed/EchoMapIntersectLengthVisitor.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -37,11 +37,12 @@ namespace Visitors {
// Collect the total number of overlapping positions
template <
- typename ProcessType,
+ typename Process,
typename BaseVisitor
>
struct EchoMapIntersectLength : BaseVisitor {
typedef BaseVisitor BaseClass;
+ typedef Process ProcessType;
typedef typename BaseClass::RefType RefType;
typedef typename BaseClass::MapType MapType;
diff --git a/interfaces/general-headers/algorithm/visitors/bed/OvrAggregateVisitor.hpp b/interfaces/general-headers/algorithm/visitors/bed/OvrAggregateVisitor.hpp
index a5c0223..caf9d68 100644
--- a/interfaces/general-headers/algorithm/visitors/bed/OvrAggregateVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/bed/OvrAggregateVisitor.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/bed/OvrUniqueFractionVisitor.hpp b/interfaces/general-headers/algorithm/visitors/bed/OvrUniqueFractionVisitor.hpp
index 828b9f7..be50d50 100644
--- a/interfaces/general-headers/algorithm/visitors/bed/OvrUniqueFractionVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/bed/OvrUniqueFractionVisitor.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/bed/OvrUniqueVisitor.hpp b/interfaces/general-headers/algorithm/visitors/bed/OvrUniqueVisitor.hpp
index 618c9eb..5536bae 100644
--- a/interfaces/general-headers/algorithm/visitors/bed/OvrUniqueVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/bed/OvrUniqueVisitor.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/numerical/AverageVisitor.hpp b/interfaces/general-headers/algorithm/visitors/bed/WeightedAverageVisitor.hpp
similarity index 60%
copy from interfaces/general-headers/algorithm/visitors/numerical/AverageVisitor.hpp
copy to interfaces/general-headers/algorithm/visitors/bed/WeightedAverageVisitor.hpp
index 6380d3b..3db69d5 100644
--- a/interfaces/general-headers/algorithm/visitors/numerical/AverageVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/bed/WeightedAverageVisitor.hpp
@@ -1,10 +1,10 @@
/*
- Author: Shane Neph & Scott Kuehn
- Date: Thu Aug 23 17:42:22 PDT 2007
+ Author: Shane Neph
+ Date: Jan.2017
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -21,8 +21,10 @@
// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
//
-#ifndef CLASS_WINDOW_AVERAGE_VISITOR_H
-#define CLASS_WINDOW_AVERAGE_VISITOR_H
+#ifndef CLASS_WINDOW_WEIGHTED_AVERAGE_VISITOR_H
+#define CLASS_WINDOW_WEIGHTED_AVERAGE_VISITOR_H
+
+#include <set>
#include "data/measurement/NaN.hpp"
@@ -32,49 +34,58 @@ namespace Visitors {
typename Process,
typename BaseVisitor
>
- struct Average : BaseVisitor {
+ struct WeightedAverage : BaseVisitor {
typedef BaseVisitor BaseClass;
typedef Process ProcessType;
typedef typename BaseClass::RefType RefType;
typedef typename BaseClass::MapType MapType;
- explicit Average(const ProcessType& pt = ProcessType())
- : pt_(pt), sum_(0), counter_(0)
+ explicit WeightedAverage(const ProcessType& pt = ProcessType())
+ : pt_(pt)
{ /* */ }
inline void Add(MapType* bt) {
- sum_ += *bt;
- ++counter_;
+ m_.insert(bt);
}
inline void Delete(MapType* bt) {
- sum_ -= *bt;
- --counter_;
+ m_.erase(bt);
}
inline void DoneReference() {
static const Signal::NaN nan = Signal::NaN();
- if ( counter_ > 0 )
- pt_.operator()(sum_/counter_);
- else
+ if ( !m_.empty() ) {
+ double value = 0;
+ double weightSum = 0;
+ for ( auto ptr : m_ ) {
+ auto v = r_->overlap(*ptr)/static_cast<double>(r_->length());
+ value += v * (*ptr);
+ weightSum += v;
+ } // for
+ value /= weightSum;
+ pt_.operator()(value);
+ } else {
pt_.operator()(nan);
+ }
}
inline void End() {
- sum_ = 0;
- counter_ = 0;
+ m_.clear();
+ r_ = nullptr;
}
- virtual ~Average()
+ inline void SetReference(RefType* r) { r_ = r; }
+
+ virtual ~WeightedAverage()
{ /* */ }
protected:
ProcessType pt_;
- double sum_;
- int counter_;
+ RefType* r_;
+ std::set<MapType*> m_;
};
} // namespace Visitors
-#endif // CLASS_WINDOW_AVERAGE_VISITOR_H
+#endif // CLASS_WINDOW_WEIGHTED_AVERAGE_VISITOR_H
diff --git a/interfaces/general-headers/algorithm/visitors/helpers/NamedVisitors.hpp b/interfaces/general-headers/algorithm/visitors/helpers/NamedVisitors.hpp
index 628185e..b1f7ffe 100644
--- a/interfaces/general-headers/algorithm/visitors/helpers/NamedVisitors.hpp
+++ b/interfaces/general-headers/algorithm/visitors/helpers/NamedVisitors.hpp
@@ -5,7 +5,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -34,6 +34,7 @@
#include "algorithm/visitors/OtherVisitors.hpp"
#include "algorithm/visitors/helpers/ProcessBedVisitorRow.hpp"
#include "algorithm/visitors/helpers/ProcessVisitorRow.hpp"
+#include "data/bed/BedCompare.hpp"
#include "utility/OrderCompare.hpp"
// Names returned via VisitorName<...>::Name() must be unique
@@ -102,8 +103,8 @@ namespace Visitors {
{ return "mad"; }
};
- template <typename A, typename B, typename C>
- struct VisitorName< Visitors::Extreme< A,B,Ordering::CompValueThenAddressGreater<C,C> > > {
+ template <typename A, typename B, typename C, typename D>
+ struct VisitorName< Visitors::Extreme< A,B,Ordering::CompValueThenAddressGreater<C,C>,D > > {
static std::string Name()
{ return "max"; }
};
@@ -120,8 +121,8 @@ namespace Visitors {
{ return "median"; }
};
- template <typename A, typename B, typename C>
- struct VisitorName< Visitors::Extreme< A,B,Ordering::CompValueThenAddressLesser<C, C> > > {
+ template <typename A, typename B, typename C, typename D>
+ struct VisitorName< Visitors::Extreme< A,B,Ordering::CompValueThenAddressLesser<C, C>,D > > {
static std::string Name()
{ return "min"; }
};
@@ -176,18 +177,6 @@ namespace Visitors {
{ return "bases-uniq-f"; }
};
- template <typename B, typename C>
- struct VisitorName< Visitors::Extreme< Visitors::BedHelpers::PrintAllScorePrecision,B,Ordering::CompValueThenAddressGreater<C,C> > > {
- static std::string Name()
- { return "max-element"; }
- };
-
- template <typename B, typename C>
- struct VisitorName< Visitors::Extreme< Visitors::BedHelpers::PrintAllScorePrecision,B,Ordering::CompValueThenAddressLesser<C, C> > > {
- static std::string Name()
- { return "min-element"; }
- };
-
template <template<class X> class A, class B>
struct VisitorName< Visitors::BedSpecific::EchoMapBed<A<Visitors::BedHelpers::Print>,B> > {
static std::string Name()
@@ -206,6 +195,18 @@ namespace Visitors {
{ return "echo-map-id-uniq"; }
};
+ template <class B>
+ struct VisitorName< Visitors::BedSpecific::EchoMapBed< Visitors::BedHelpers::PrintGenomicRange<Visitors::BedHelpers::PrintBED3>, B> > {
+ static std::string Name()
+ { return "echo-map-range"; }
+ };
+
+ template <template<class X> class A, class B>
+ struct VisitorName< Visitors::BedSpecific::EchoMapBed< A<Visitors::BedHelpers::PrintScorePrecision>, B> > {
+ static std::string Name()
+ { return "echo-map-score"; }
+ };
+
template <template<class X> class A, class B>
struct VisitorName< Visitors::BedSpecific::EchoMapBed<A<Visitors::BedHelpers::PrintLength>,B> > {
static std::string Name()
@@ -218,16 +219,34 @@ namespace Visitors {
{ return "echo-overlap-size"; }
};
- template <class B>
- struct VisitorName< Visitors::BedSpecific::EchoMapBed< Visitors::BedHelpers::PrintGenomicRange<Visitors::BedHelpers::PrintBED3>, B> > {
+ template <typename B, typename C, typename D>
+ struct VisitorName< Visitors::Extreme< Visitors::BedHelpers::PrintAllScorePrecision,B,Bed::ScoreThenGenomicCompareGreater<C,C>, D > > {
static std::string Name()
- { return "echo-map-range"; }
+ { return "max-element"; }
};
- template <template<class X> class A, class B>
- struct VisitorName< Visitors::BedSpecific::EchoMapBed< A<Visitors::BedHelpers::PrintScorePrecision>, B> > {
+ template <typename B, typename C, typename D>
+ struct VisitorName< Visitors::Extreme< Visitors::BedHelpers::PrintAllScorePrecision,B,Ordering::CompValueThenAddressGreater<C,C>, D > > {
static std::string Name()
- { return "echo-map-score"; }
+ { return "max-element-rand"; }
+ };
+
+ template <typename B, typename C, typename D>
+ struct VisitorName< Visitors::Extreme< Visitors::BedHelpers::PrintAllScorePrecision,B,Bed::ScoreThenGenomicCompareLesser<C,C>, D > > {
+ static std::string Name()
+ { return "min-element"; }
+ };
+
+ template <typename B, typename C, typename D>
+ struct VisitorName< Visitors::Extreme< Visitors::BedHelpers::PrintAllScorePrecision,B,Ordering::CompValueThenAddressLesser<C, C>, D > > {
+ static std::string Name()
+ { return "min-element-rand"; }
+ };
+
+ template <typename A, typename B>
+ struct VisitorName< Visitors::WeightedAverage<A,B> > {
+ static std::string Name()
+ { return "wmean"; }
};
} // namespace Helpers
diff --git a/interfaces/general-headers/algorithm/visitors/helpers/ProcessBedVisitorRow.hpp b/interfaces/general-headers/algorithm/visitors/helpers/ProcessBedVisitorRow.hpp
index d092c99..f73c628 100644
--- a/interfaces/general-headers/algorithm/visitors/helpers/ProcessBedVisitorRow.hpp
+++ b/interfaces/general-headers/algorithm/visitors/helpers/ProcessBedVisitorRow.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/helpers/ProcessVisitorRow.hpp b/interfaces/general-headers/algorithm/visitors/helpers/ProcessVisitorRow.hpp
index b4fcbfc..9236a5d 100644
--- a/interfaces/general-headers/algorithm/visitors/helpers/ProcessVisitorRow.hpp
+++ b/interfaces/general-headers/algorithm/visitors/helpers/ProcessVisitorRow.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/numerical/AverageVisitor.hpp b/interfaces/general-headers/algorithm/visitors/numerical/AverageVisitor.hpp
index 6380d3b..ae65383 100644
--- a/interfaces/general-headers/algorithm/visitors/numerical/AverageVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/numerical/AverageVisitor.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/numerical/CoeffVariationVisitor.hpp b/interfaces/general-headers/algorithm/visitors/numerical/CoeffVariationVisitor.hpp
index bebc298..dbbb638 100644
--- a/interfaces/general-headers/algorithm/visitors/numerical/CoeffVariationVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/numerical/CoeffVariationVisitor.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/numerical/CountVisitor.hpp b/interfaces/general-headers/algorithm/visitors/numerical/CountVisitor.hpp
index 7a649d9..235e7a2 100644
--- a/interfaces/general-headers/algorithm/visitors/numerical/CountVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/numerical/CountVisitor.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/numerical/ExtremeVisitor.hpp b/interfaces/general-headers/algorithm/visitors/numerical/ExtremeVisitor.hpp
index d9c9de1..f9c0fdc 100644
--- a/interfaces/general-headers/algorithm/visitors/numerical/ExtremeVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/numerical/ExtremeVisitor.hpp
@@ -5,7 +5,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -25,9 +25,12 @@
#ifndef CLASS_WINDOW_EXTREME_VISITOR_H
#define CLASS_WINDOW_EXTREME_VISITOR_H
-#include <iostream>
+#include <algorithm>
+#include <cstdlib>
+#include <ctime>
#include <set>
#include <string>
+#include <type_traits>
#include "data/measurement/NaN.hpp"
#include "utility/OrderCompare.hpp"
@@ -39,13 +42,44 @@ namespace Visitors {
not a std::multiset<>. Use a version of the templated
CompValueThenAddress<> or a similar idea.
*/
+ struct DoNothing {};
+ struct RandTie {
+ RandTie() {
+ std::srand(std::time(NULL));
+ }
+
+ template <typename T, typename C>
+ T* breakTie(const std::set<T*, C>& s) {
+ // s is not empty if Extreme is calling
+ std::vector<T*> toRand;
+ bool first = true;
+ double best = 0;
+ for ( auto iter = s.begin(); iter != s.end(); ++iter ) {
+ if ( first ) {
+ best = **iter;
+ toRand.push_back(*iter);
+ first = false;
+ } else if ( **iter == best ) {
+ toRand.push_back(*iter);
+ } else {
+ break;
+ }
+ } // for
+ if ( toRand.size() == 1 )
+ return toRand.back();
+ std::random_shuffle(toRand.begin(), toRand.end());
+ return toRand.back();
+ }
+ };
+
template <
typename Process,
typename BaseVisitor,
typename CompType = Ordering::CompValueThenAddressLesser<
typename BaseVisitor::mapping_type,
typename BaseVisitor::mapping_type
- >
+ >,
+ typename OnTies = DoNothing
>
struct Extreme : BaseVisitor {
@@ -64,17 +98,36 @@ namespace Visitors {
m_.erase(bt);
}
- inline void DoneReference() {
+ void DoneReference() {
+ doneReference();
+ }
+
+ virtual ~Extreme() { /* */ }
+
+ private:
+
+ template <typename OT=OnTies>
+ inline typename std::enable_if<!std::is_same<OT, DoNothing>::value>::type
+ doneReference() {
+ static const Signal::NaN nan = Signal::NaN();
+ static OnTies onTies;
+ if ( !m_.empty() )
+ pt_.operator()(onTies.breakTie(m_));
+ else
+ pt_.operator()(nan);
+ }
+
+
+ template <typename OT=OnTies>
+ inline typename std::enable_if<std::is_same<OT, DoNothing>::value>::type
+ doneReference() {
static const Signal::NaN nan = Signal::NaN();
- if ( !m_.empty() ) {
+ if ( !m_.empty() )
pt_.operator()(*m_.begin());
- }
else
pt_.operator()(nan);
}
- virtual ~Extreme() { /* */ }
-
protected:
ProcessType pt_;
std::set<MapType*, CompType> m_;
diff --git a/interfaces/general-headers/algorithm/visitors/numerical/IndicatorVisitor.hpp b/interfaces/general-headers/algorithm/visitors/numerical/IndicatorVisitor.hpp
index c68eae1..90d76d4 100644
--- a/interfaces/general-headers/algorithm/visitors/numerical/IndicatorVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/numerical/IndicatorVisitor.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/numerical/MedianAbsoluteDeviationVisitor.hpp b/interfaces/general-headers/algorithm/visitors/numerical/MedianAbsoluteDeviationVisitor.hpp
index 21b02ba..164b142 100644
--- a/interfaces/general-headers/algorithm/visitors/numerical/MedianAbsoluteDeviationVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/numerical/MedianAbsoluteDeviationVisitor.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/numerical/MedianVisitor.hpp b/interfaces/general-headers/algorithm/visitors/numerical/MedianVisitor.hpp
index 441169a..37373ea 100644
--- a/interfaces/general-headers/algorithm/visitors/numerical/MedianVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/numerical/MedianVisitor.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/numerical/RollingKthAverageVisitor.hpp b/interfaces/general-headers/algorithm/visitors/numerical/RollingKthAverageVisitor.hpp
index 03e74b4..9c5e1b7 100644
--- a/interfaces/general-headers/algorithm/visitors/numerical/RollingKthAverageVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/numerical/RollingKthAverageVisitor.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/numerical/RollingKthVisitor.hpp b/interfaces/general-headers/algorithm/visitors/numerical/RollingKthVisitor.hpp
index 0f8c695..4c0ed2c 100644
--- a/interfaces/general-headers/algorithm/visitors/numerical/RollingKthVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/numerical/RollingKthVisitor.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/numerical/StdevVisitor.hpp b/interfaces/general-headers/algorithm/visitors/numerical/StdevVisitor.hpp
index 9cda217..05d6b2d 100644
--- a/interfaces/general-headers/algorithm/visitors/numerical/StdevVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/numerical/StdevVisitor.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/numerical/SumVisitor.hpp b/interfaces/general-headers/algorithm/visitors/numerical/SumVisitor.hpp
index c0d6d95..8b66cec 100644
--- a/interfaces/general-headers/algorithm/visitors/numerical/SumVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/numerical/SumVisitor.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/numerical/TrimmedMeanVisitor.hpp b/interfaces/general-headers/algorithm/visitors/numerical/TrimmedMeanVisitor.hpp
index 83a0e45..b1d0a2b 100644
--- a/interfaces/general-headers/algorithm/visitors/numerical/TrimmedMeanVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/numerical/TrimmedMeanVisitor.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/numerical/VarianceVisitor.hpp b/interfaces/general-headers/algorithm/visitors/numerical/VarianceVisitor.hpp
index 52a9fca..8db26e8 100644
--- a/interfaces/general-headers/algorithm/visitors/numerical/VarianceVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/numerical/VarianceVisitor.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/other/EchoVisitor.hpp b/interfaces/general-headers/algorithm/visitors/other/EchoVisitor.hpp
index d44319c..10cde93 100644
--- a/interfaces/general-headers/algorithm/visitors/other/EchoVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/other/EchoVisitor.hpp
@@ -8,7 +8,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/algorithm/visitors/other/MultiVisitor.hpp b/interfaces/general-headers/algorithm/visitors/other/MultiVisitor.hpp
index ed35378..d32b68e 100644
--- a/interfaces/general-headers/algorithm/visitors/other/MultiVisitor.hpp
+++ b/interfaces/general-headers/algorithm/visitors/other/MultiVisitor.hpp
@@ -8,7 +8,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/data/bed/AllocateIterator_BED_starch.hpp b/interfaces/general-headers/data/bed/AllocateIterator_BED_starch.hpp
index 59deedd..8b87a53 100644
--- a/interfaces/general-headers/data/bed/AllocateIterator_BED_starch.hpp
+++ b/interfaces/general-headers/data/bed/AllocateIterator_BED_starch.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -80,6 +80,7 @@ namespace Bed {
throw(ErrorType("Error: stat() failed on: " + fp.Name()));
is_namedpipe = (S_ISFIFO(st.st_mode) != 0);
}
+ is_starch_ = (is_starch_ && !is_namedpipe);
if ( (fp_ == stdin || is_namedpipe) && !all_ ) { // BED, chrom-specific, using stdin
// stream through until we find what we want
@@ -223,7 +224,7 @@ namespace Bed {
bool _M_ok;
char chr_[Bed::MAXCHROMSIZE+1];
BedType* _M_value;
- const bool is_starch_;
+ bool is_starch_;
const bool all_;
starch::Starch* archive_;
};
diff --git a/interfaces/general-headers/data/bed/Bed.hpp b/interfaces/general-headers/data/bed/Bed.hpp
index 8e2ecb6..809d204 100644
--- a/interfaces/general-headers/data/bed/Bed.hpp
+++ b/interfaces/general-headers/data/bed/Bed.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -158,7 +158,7 @@ namespace Bed {
return start_ - a.start_;
return std::numeric_limits<CoordType>::max();
}
- inline CoordType sepDistance(const BasicCoords& a) const {
+ inline SignedCoordType sepDistance(const BasicCoords& a) const {
if( 0 == std::strcmp(chrom_, a.chrom_) )
return end_ - a.start_;
return std::numeric_limits<CoordType>::max();
@@ -298,6 +298,7 @@ namespace Bed {
// Properties
char const* rest() const { return rest_; }
+ char const* full_rest() const { return rest(); }
// Operators
BasicCoords& operator=(const BasicCoords& c) {
@@ -361,7 +362,6 @@ namespace Bed {
~BasicCoords() {
if ( rest_ )
delete [] rest_;
- rest_ = NULL;
}
static const bool UseRest = true;
@@ -505,26 +505,36 @@ namespace Bed {
struct Bed4
: public Bed4<BedType, false> {
- Bed4() : BaseClass(), rest_(new char[1]) { *rest_ = '\0'; }
+ Bed4() : BaseClass(), rest_(new char[1]), fullrest_(new char[1]) { *rest_ = '\0'; *fullrest_ = '\0'; }
Bed4(const Bed4& c)
- : BaseClass(c), rest_(new char[(c.rest_ != NULL) ? (std::strlen(c.rest_)+1) : 1])
- { *rest_ = '\0'; if ( c.rest_ != NULL ) std::strcpy(rest_, c.rest_); }
+ : BaseClass(c), rest_(new char[(c.rest_ != NULL) ? (std::strlen(c.rest_)+1) : 1]),
+ fullrest_(new char[(c.fullrest_ != NULL) ? (std::strlen(c.fullrest_)+1) : 1])
+ { *rest_ = '\0'; *fullrest_ = '\0';
+ if ( c.rest_ != NULL ) std::strcpy(rest_, c.rest_);
+ if ( c.fullrest_ != NULL ) std::strcpy(fullrest_, c.fullrest_);
+ }
Bed4(char const* chrom, CoordType start, CoordType end, char const* id, char const* rest = NULL)
- : BaseClass(chrom, start, end, id), rest_(new char[(rest != NULL) ? (std::strlen(rest)+1) : 1]) {
- *rest_ = '\0';
+ : BaseClass(chrom, start, end, id), rest_(new char[(rest != NULL) ? (std::strlen(rest)+1) : 1]),
+ fullrest_(new char[((rest != NULL) ? (std::strlen(rest)+1) : 1) + ((id != NULL) ? (std::strlen(id)+1) : 1)]) {
+ *rest_ = '\0'; *fullrest_ = '\0';
+ if ( id && std::strcmp(id, "") != 0 )
+ std::strcpy(fullrest_, id);
+
if ( rest && std::strcmp(rest, "") != 0 ) {
if ( rest[0] != '\t' )
rest_[0] = '\t';
std::strcat(rest_, rest);
+ std::strcat(fullrest_, rest_);
}
}
- explicit Bed4(FILE* inF) : BaseClass(), rest_(0)
+ explicit Bed4(FILE* inF) : BaseClass(), rest_(0), fullrest_(0)
{ this->readline(inF); }
- explicit Bed4(const std::string& inS) : BaseClass(), rest_(0)
+ explicit Bed4(const std::string& inS) : BaseClass(), rest_(0), fullrest_(0)
{ this->readline(inS); }
// Properties
char const* rest() const { return rest_; }
+ char const* full_rest() const { return fullrest_; }
// IO
inline void print() const {
@@ -556,6 +566,12 @@ namespace Bed {
delete [] rest_;
rest_ = new char[std::strlen(restBuf) + 1];
std::strcpy(rest_, restBuf);
+ if ( fullrest_ )
+ delete [] fullrest_;
+ std::size_t sz = (std::strlen(restBuf)+1) + (std::strlen(idBuf) + 1);
+ fullrest_ = new char[sz];
+ std::strcpy(fullrest_, idBuf);
+ std::strcat(fullrest_, restBuf);
return numScanned;
}
inline int readline(FILE* inputFile) {
@@ -580,6 +596,12 @@ namespace Bed {
delete [] rest_;
rest_ = new char[std::strlen(restBuf) + 1];
std::strcpy(rest_, restBuf);
+ std::size_t sz = (std::strlen(restBuf)+1) + (std::strlen(idBuf) + 1);
+ if ( fullrest_ )
+ delete [] fullrest_;
+ fullrest_ = new char[sz];
+ std::strcpy(fullrest_, idBuf);
+ std::strcat(fullrest_, restBuf);
return numScanned;
}
@@ -588,15 +610,20 @@ namespace Bed {
BaseClass::operator=(c);
if ( rest_ )
delete [] rest_;
+ if ( fullrest_ )
+ delete [] fullrest_;
rest_ = new char[(c.rest_ != NULL) ? (std::strlen(c.rest_) + 1) : 1];
*rest_ = '\0'; if ( c.rest_ != NULL ) std::strcpy(rest_, c.rest_);
+ fullrest_ = new char[(c.fullrest_ != NULL) ? (std::strlen(c.fullrest_) + 1) : 1];
+ *fullrest_ = '\0'; if ( c.fullrest_ != NULL ) std::strcpy(fullrest_, c.fullrest_);
return *this;
}
~Bed4() {
if ( rest_ )
delete [] rest_;
- rest_ = NULL;
+ if ( fullrest_ )
+ delete [] fullrest_;
}
static const bool UseRest = true;
@@ -609,6 +636,7 @@ namespace Bed {
using BaseClass::id_;
char* rest_;
+ char* fullrest_;
static std::string outFormatter() {
return(BaseClass::outFormatter() + "%s");
}
@@ -727,26 +755,37 @@ namespace Bed {
Bed5() : BaseClass(), rest_(new char[1]) { *rest_ = '\0'; }
Bed5(const Bed5& c)
- : BaseClass(c), rest_(new char[(c.rest_ != NULL) ? (std::strlen(c.rest_+1)) : 1])
- { *rest_ = '\0'; if ( c.rest_ != NULL ) std::strcpy(rest_, c.rest_); }
+ : BaseClass(c), rest_(new char[(c.rest_ != NULL) ? (std::strlen(c.rest_+1)) : 1]),
+ fullrest_(new char[(c.fullrest_ != NULL) ? (std::strlen(c.fullrest_+1)) : 1])
+ {
+ *rest_ = '\0'; if ( c.rest_ != NULL ) std::strcpy(rest_, c.rest_);
+ *fullrest_ = '\0'; if ( c.fullrest_ != NULL ) std::strcpy(fullrest_, c.fullrest_);
+ }
Bed5(char const* chrom, CoordType start, CoordType end,
char const* id, MeasureType measurement, char const* rest = NULL)
: BaseClass(chrom, start, end, id, measurement),
- rest_(new char[(rest != NULL) ? (std::strlen(rest)+1) : 1]) {
- *rest_ = '\0';
+ rest_(new char[(rest != NULL) ? (std::strlen(rest)+1) : 1]),
+ fullrest_(new char[((rest != NULL) ? (std::strlen(rest)+1) : 1) + ((id != NULL) ? (std::strlen(id)+1) : 1)])
+ {
+ *rest_ = '\0'; *fullrest_ = '\0';
+ if ( id && std::strcmp(id, "") != 0 )
+ std::strcpy(fullrest_, id);
+
if ( rest && 0 != std::strcmp(rest, "") ) {
if ( rest[0] != '\t' )
rest_[0] = '\t';
std::strcat(rest_, rest);
+ std::strcat(fullrest_, rest_);
}
}
- explicit Bed5(FILE* inF) : BaseClass(), rest_(0)
+ explicit Bed5(FILE* inF) : BaseClass(), rest_(0), fullrest_(0)
{ this->readline(inF); }
- explicit Bed5(const std::string& inS) : BaseClass(), rest_(0)
+ explicit Bed5(const std::string& inS) : BaseClass(), rest_(0), fullrest_(0)
{ this->readline(inS); }
// Properties
char const* rest() const { return rest_; }
+ char const* full_rest() const { return fullrest_; }
// IO
inline void print() const {
@@ -779,6 +818,12 @@ namespace Bed {
delete [] rest_;
rest_ = new char[std::strlen(restBuf) + 1];
std::strcpy(rest_, restBuf);
+ if ( fullrest_ )
+ delete [] fullrest_;
+ std::size_t sz = (std::strlen(restBuf)+1) + (std::strlen(idBuf) + 1);
+ fullrest_ = new char[sz];
+ std::strcpy(fullrest_, idBuf);
+ std::strcat(fullrest_, restBuf);
return numScanned;
}
inline int readline(FILE* inputFile) {
@@ -803,6 +848,14 @@ namespace Bed {
delete [] rest_;
rest_ = new char[std::strlen(restBuf) + 1];
std::strcpy(rest_, restBuf);
+
+ if ( fullrest_ )
+ delete [] fullrest_;
+ std::size_t sz = (std::strlen(restBuf)+1) + (std::strlen(idBuf) + 1);
+ fullrest_ = new char[sz];
+ std::strcpy(fullrest_, idBuf);
+ std::strcat(fullrest_, restBuf);
+
return numScanned;
}
@@ -811,15 +864,20 @@ namespace Bed {
BaseClass::operator=(c);
if ( rest_ )
delete [] rest_;
+ if ( fullrest_ )
+ delete [] fullrest_;
rest_ = new char[(c.rest_ != NULL) ? (std::strlen(c.rest_) + 1) : 1];
*rest_ = '\0'; if ( c.rest_ != NULL ) std::strcpy(rest_, c.rest_);
+ fullrest_ = new char[(c.fullrest_ != NULL) ? (std::strlen(c.fullrest_) + 1) : 1];
+ *fullrest_ = '\0'; if ( c.fullrest_ != NULL ) std::strcpy(fullrest_, c.fullrest_);
return *this;
}
~Bed5() {
if ( rest_ )
delete [] rest_;
- rest_ = NULL;
+ if ( fullrest_ )
+ delete [] fullrest_;
}
// Parameters
@@ -834,242 +892,7 @@ namespace Bed {
using BaseClass::measurement_;
char* rest_;
-
- static std::string outFormatter() {
- return(BaseClass::outFormatter() + "%s");
- }
-
- static std::string inFormatter() {
- return(BaseClass::outFormatter() + "%[^\n]s\n");
- }
- };
-
-
- /*****************************************/
- /* Bed6 Classes */
- /*****************************************/
-
- // Forward declaration, forcing specialization
- template <typename Bed5Type, bool HasRest = false>
- struct Bed6;
-
- // Bed6 specialization 1: vanilla (default) information
- template <typename U, typename MeasureType, bool Bed5HasRest>
- struct Bed6<Bed5<U, MeasureType, Bed5HasRest>, false>
- : public Bed5<U, MeasureType, false> {
-
- Bed6() : BaseClass(), strand_(Bed::PLUS) {}
- Bed6(const Bed6& c) : BaseClass(c), strand_(c.strand_) {}
- Bed6(char const* chrom, CoordType start, CoordType end,
- char const* id, MeasureType measurement, Strand strand)
- : BaseClass(chrom, start, end, id, measurement), strand_(strand) {}
- explicit Bed6(FILE* inF) : BaseClass(), strand_(Bed::PLUS) { this->readline(inF); }
- explicit Bed6(const std::string& inS) : BaseClass(), strand_(Bed::PLUS)
- { this->readline(inS); }
-
- // Properties
- CoordType distance(const Bed6& a) {
- if ( strand_ != a.strand_ )
- return std::numeric_limits< CoordType >::max();
- return U::distance(a);
- }
- CoordType sepDistance(const Bed6& a) {
- if ( strand_ != a.strand_ )
- return std::numeric_limits<CoordType>::max();
- return U::sepDistance(a);
- }
- void strand(Strand strand) { strand_ = strand; }
- Strand strand() const { return strand_; }
-
- // IO
- inline int readline(const std::string& inputLine) {
- static char chrBuf[MAXCHROMSIZE + 1];
- chrBuf[0] = '\0';
- static char idBuf[MAXIDSIZE + 1];
- idBuf[0] = '\0';
-
- static const std::string lclStatic = inFormatter();
- static char const* format = lclStatic.c_str();
- int numScanned = std::sscanf(inputLine.c_str(), format,
- chrBuf,
- &start_,
- &end_,
- idBuf);
- this->chrom(chrBuf);
- this->id(idBuf);
- return numScanned;
- }
- inline int readline(FILE* inputFile) {
- static char chrBuf[MAXCHROMSIZE + 1];
- chrBuf[0] = '\0';
- static char idBuf[MAXIDSIZE + 1];
- idBuf[0] = '\0';
-
- static const std::string lclStatic = inFormatter();
- static char const* format = lclStatic.c_str();
- int numScanned = std::fscanf(inputFile, format,
- chrBuf,
- &start_,
- &end_,
- idBuf);
- std::fgetc(inputFile); // read and discard newline
- this->chrom(chrBuf);
- this->id(idBuf);
- return numScanned;
- }
- inline void print() const {
- static const std::string lclStatic = outFormatter();
- static char const* format = lclStatic.c_str();
- std::printf(format, chrom_, start_, end_, id_, measurement_, strand_);
- }
- inline void println() const {
- static const std::string heapFormat = (outFormatter() + "\n");
- static char const* format = heapFormat.c_str();
- std::printf(format, chrom_, start_, end_, id_, measurement_, strand_);
- }
-
- // Operators
- Bed6& operator=(const Bed6& c) {
- BaseClass::operator=(c);
- this->strand_ = c.strand_;
- return *this;
- }
-
- static const int NumFields = 6;
- static const bool UseRest = false;
-
- protected:
- typedef Bed5<U, MeasureType, false> BaseClass;
- using BaseClass::chrom_;
- using BaseClass::start_;
- using BaseClass::end_;
- using BaseClass::id_;
- using BaseClass::measurement_;
-
- Strand strand_;
-
- static std::string outFormatter() {
- return(BaseClass::outFormatter() + "\t%c");
- }
-
- static std::string inFormatter() {
- return(outFormatter() + "%*[^\n]s\n");
- }
- };
-
- // Specialization 2: Extend specialization 1 with "rest-size" information
- template <typename U, bool HasRest>
- struct Bed6
- : public Bed6<U, false> { /* U is forced to be Bed5<> by single specialization above */
-
- Bed6() : BaseClass(), rest_(new char[1]) { *rest_ = '\0'; }
- Bed6(const Bed6& c) : BaseClass(c), rest_(new char[(c.rest_ != NULL) ? (std::strlen(c.rest_)+1) : 1])
- { *rest_ = '\0'; if ( c.rest_ != NULL ) std::strcpy(rest_, c.rest_); }
- Bed6(char const* chrom, CoordType start, CoordType end,
- char const* id, typename U::MeasurementType measurement,
- Strand strand, char const* rest = NULL)
- : BaseClass(chrom, start, end, id, measurement, strand),
- rest_(new char[(rest != NULL) ? (std::strlen(rest)+1) : 1]) {
- *rest_ = '\0';
- if ( rest && std::strcmp(rest, "") != 0 ) {
- if ( rest[0] != '\t' )
- rest_[0] = '\t';
- std::strcat(rest_, rest);
- }
- }
- explicit Bed6(FILE* inF) : BaseClass(), rest_(0)
- { this->readline(inF); }
- explicit Bed6(const std::string& inS) : BaseClass(), rest_(0)
- { this->readline(inS); }
-
- // Properties
- char const* rest() const { return rest_; }
-
- // IO
- inline int readline(const std::string& inputLine) {
- static char chrBuf[MAXCHROMSIZE + 1];
- chrBuf[0] = '\0';
- static char idBuf[MAXIDSIZE + 1];
- idBuf[0] = '\0';
- static char restBuf[MAXRESTSIZE + 1];
- restBuf[0] = '\0';
- static const std::string lclStatic = inFormatter();
- static char const* format = lclStatic.c_str();
- int numScanned = std::sscanf(inputLine.c_str(),
- format, chrBuf,
- &start_, &end_, idBuf,
- &measurement_, &strand_, restBuf);
- this->chrom(chrBuf);
- this->id(idBuf);
- if ( rest_ )
- delete [] rest_;
- rest_ = new char[std::strlen(restBuf) + 1];
- std::strcpy(rest_, restBuf);
- return numScanned;
- }
- inline int readline(FILE* inputFile) {
- static char chrBuf[MAXCHROMSIZE + 1];
- chrBuf[0] = '\0';
- static char idBuf[MAXIDSIZE + 1];
- idBuf[0] = '\0';
- static char restBuf[MAXRESTSIZE + 1];
- restBuf[0] = '\0';
-
- static const std::string lclStatic = inFormatter();
- static char const* format = lclStatic.c_str();
- int numScanned = std::fscanf(inputFile,
- format, chrBuf,
- &start_, &end_, idBuf,
- &measurement_, &strand_, restBuf);
- std::fgetc(inputFile); // Read and discard trailing newline
- this->chrom(chrBuf);
- this->id(idBuf);
- if ( rest_ )
- delete [] rest_;
- rest_ = new char[std::strlen(restBuf) + 1];
- std::strcpy(rest_, restBuf);
- return numScanned;
- }
- inline void print() const {
- static const std::string lclStatic = outFormatter();
- static char const* format = lclStatic.c_str();
- std::printf(format, chrom_, start_, end_, id_, measurement_, strand_, rest_);
- }
- inline void println() const {
- static const std::string heapFormat = outFormatter() + "\n";
- static char const* format = heapFormat.c_str();
- std::printf(format, chrom_, start_, end_, id_, measurement_, strand_, rest_);
- }
-
- // Operators
- Bed6& operator=(const Bed6& c) {
- BaseClass::operator=(c);
- if ( rest_ )
- delete [] rest_;
- rest_ = new char[(c.rest_ != NULL) ? (std::strlen(c.rest_) + 1) : 1];
- *rest_ = '\0';
- if ( c.rest_ != NULL ) std::strcpy(rest_, c.rest_);
- return *this;
- }
-
- ~Bed6() {
- if ( rest_ )
- delete [] rest_;
- rest_ = NULL;
- }
-
- static const bool UseRest = true;
-
- protected:
- typedef Bed6<U, false> BaseClass;
- using BaseClass::chrom_;
- using BaseClass::start_;
- using BaseClass::end_;
- using BaseClass::id_;
- using BaseClass::measurement_;
- using BaseClass::strand_;
-
- char* rest_;
+ char* fullrest_;
static std::string outFormatter() {
return(BaseClass::outFormatter() + "%s");
diff --git a/interfaces/general-headers/data/bed/BedCheckIterator.hpp b/interfaces/general-headers/data/bed/BedCheckIterator.hpp
index 88e6dab..c927ed9 100644
--- a/interfaces/general-headers/data/bed/BedCheckIterator.hpp
+++ b/interfaces/general-headers/data/bed/BedCheckIterator.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -27,11 +27,13 @@
#include <cctype>
#include <cstddef>
#include <cstdlib>
+#include <fstream>
#include <istream>
#include <iterator>
#include <limits>
#include <sstream>
#include <string>
+#include <type_traits>
#include <vector>
#include <sys/stat.h>
@@ -60,16 +62,16 @@ namespace Bed {
typedef BedType** pointer;
typedef BedType*& reference;
- static const int nFields_ = BedType::NumFields;
- static const bool hasRest_ = BedType::UseRest;
+ static constexpr int nFields_ = BedType::NumFields;
+ static constexpr bool hasRest_ = BedType::UseRest;
bed_check_iterator() : fp_(std::cin), _M_ok(false), _M_value(0), fn_(""), cnt_(0),
- lastChr_(""), lastStart_(1), lastEnd_(0), nestCheck_(false),
+ lastChr_(""), lastRest_(""), lastStart_(1), lastEnd_(0), nestCheck_(false),
maxEnd_(0), chr_(""), isStarch_(false), all_(true), archive_(0)
{ /* */ }
bed_check_iterator(std::istream& is, const std::string& filename, const std::string& chr = "all", bool nestCheck = false)
- : fp_(is), _M_ok(fp_), _M_value(0), fn_(filename), cnt_(0), lastChr_(""), lastStart_(1),
+ : fp_(is), _M_ok(fp_), _M_value(0), fn_(filename), cnt_(0), lastChr_(""), lastRest_(""), lastStart_(1),
lastEnd_(0), nestCheck_(nestCheck), maxEnd_(0), chr_(chr),
isStarch_(false), all_(chr_ == "all"),
archive_(0) {
@@ -86,6 +88,8 @@ namespace Bed {
}
isStarch_ = (fp_ && !is_namedpipe && (&is != &std::cin) && starch::Starch::isStarch(fn_));
+ if ( isStarch_ ) // Starch constructor opens a stream for us
+ dynamic_cast<std::ifstream&>(fp_).close(); // isStarch_ ensures fp_ is open and it's not std::cin/named pipe
// compare pointers directly, to allow compilation with Clang/LLVM against C++11 standard
if ( (&fp_ == &std::cin || is_namedpipe) && !all_ ) { // only BED through stdin; chromosome-specific
@@ -305,18 +309,18 @@ namespace Bed {
std::string t(s);
for ( std::size_t i = 0; i < t.size(); ++i )
t[i] = static_cast<char>(std::tolower(t[i]));
- return(t);
+ return t;
}
bool isUCSCheader(const std::string& bl) {
std::string tmp = lowerstr(bl);
- return(tmp == "browser" || tmp == "track");
+ return (tmp == "browser" || tmp == "track");
}
bool get_starch(std::string& line) {
if ( archive_ == NULL || !archive_->extractBEDLine(line) )
- return(false);
- return(!line.empty());
+ return false;
+ return !line.empty();
}
bool check(const std::string& bl) {
@@ -327,28 +331,28 @@ namespace Bed {
if ( bl.empty() )
msg = "Empty line found.";
else if ( isUCSCheader(bl) )
- return(false);
+ return false;
// Check chromosome
std::string::size_type marker = 0, sz = bl.size();
while ( msg.empty() && marker < sz ) {
if ( bl[marker] == ' ' ) {
if ( isUCSCheader(bl.substr(0, marker)) )
- return(false);
+ return false;
msg = "First column should not have spaces. Consider 'chr1' vs. 'chr1 '. These are different names.\nsort-bed can correct this for you.";
}
else if ( 0 == marker && bl[marker] == '@' ) {
- return(false); // SAM format header supported by starch
+ return false; // SAM format header supported by starch
}
else if ( 0 == marker && bl[marker] == '#' ) {
- return(false); // VCF format header supported by starch
+ return false; // VCF format header supported by starch
}
else if ( bl[marker] == '\t' ) {
if ( 0 == marker )
msg = "First column name should not start with a tab.";
else {
if ( isUCSCheader(bl.substr(0, marker)) )
- return(false);
+ return false;
break;
}
}
@@ -391,7 +395,7 @@ namespace Bed {
}
++marker;
} // while
-
+
if ( msg.empty() ) {
if ( sz <= marker )
msg = "No tabs after start coordinate.";
@@ -406,10 +410,10 @@ namespace Bed {
++marker; // increment passed tab
}
}
-
-
+
// Check end coordinate
pos = marker;
+ auto restMarker = marker;
while ( msg.empty() && marker < sz ) {
if ( !std::isdigit(bl[marker]) ) {
if ( bl[marker] == '\t' && pos != marker )
@@ -427,7 +431,7 @@ namespace Bed {
}
++marker;
} // while
-
+
if ( msg.empty() ) {
if ( sz <= marker && nFields_ > 3 ) {
std::stringstream con; con << nFields_;
@@ -444,8 +448,7 @@ namespace Bed {
++marker; // increment passed tab
}
}
-
-
+
if ( nFields_ > 3 ) { // check ID field
pos = marker;
while ( msg.empty() && marker < sz ) {
@@ -458,7 +461,7 @@ namespace Bed {
++marker;
} // while
-
+
if ( msg.empty() ) {
if ( sz <= marker && nFields_ > 4 ) {
std::stringstream con; con << nFields_;
@@ -477,8 +480,7 @@ namespace Bed {
else
++marker; // increment passed tab
}
-
-
+
if ( nFields_ > 4 ) { // check measurement column
pos = marker;
static int decimalCount = 0;
@@ -510,12 +512,12 @@ namespace Bed {
}
else if ( bl[marker] == ' ' )
msg = "Measurement value may not contain a space.";
- else if ( bl[marker] == '-' ) {
+ else if ( bl[marker] == '-' || bl[marker] == '+' ) {
if ( marker != pos && expCount < 1 )
- msg = "Measurement value has '-' in wrong place.";
+ msg = "Measurement value has '-' or '+' in wrong place.";
if ( msg.empty() && marker != pos ) {
if ( ++minusCount > 1 )
- msg = "Measurement value has multiple '-' characters.";
+ msg = "Measurement value has multiple '-' and/or '+' characters.";
else if ( expPos + 1 != marker )
msg = "Measurement value has bad '-' in the exponent.";
minusPos = marker;
@@ -589,7 +591,7 @@ namespace Bed {
s << cnt_;
throw(Exception("in " + fn_ + "\n" + msg + "\nSee row: " + s.str()));
}
-
+
_M_value = new BedType(bl);
if ( msg.empty() && !lastChr_.empty() ) {
cmp = std::strcmp(_M_value->chrom(), lastChr_.c_str());
@@ -598,16 +600,22 @@ namespace Bed {
else if ( cmp == 0 ) {
if ( _M_value->start() < lastStart_ )
msg = "Bed file not properly sorted by start coordinates.";
- else if ( _M_value->start() == lastStart_ && _M_value->end() < lastEnd_ )
- msg = "Bed file not properly sorted by end coordinates when start coordinates are identical.";
-
+ else if ( _M_value->start() == lastStart_ ) {
+ if ( _M_value->end() < lastEnd_ )
+ msg = "Bed file not properly sorted by end coordinates when start coordinates are identical.";
+ else if ( hasRest_ && _M_value->end() == lastEnd_ ) {
+ if ( std::strcmp(bl.substr(restMarker).c_str(), lastRest_.c_str()) < 0 )
+ msg = "Bed file not sorted by information following the 3rd column (columns 1-3 equal to previous row).";
+ }
+ }
+
if ( msg.empty() && nestCheck_ ) { // _M_value->start() > lastStart_
if ( _M_value->end() < maxEnd_ )
msg = "Fully nested component found.";
}
}
}
-
+
if ( msg.empty() && _M_value->end() <= _M_value->start() )
msg = "End coordinates must be greater than start coordinates.";
@@ -620,18 +628,18 @@ namespace Bed {
lastChr_ = _M_value->chrom();
lastStart_ = _M_value->start();
lastEnd_ = _M_value->end();
+ lastRest_ = bl.substr(restMarker);
maxEnd_ = lastEnd_;
- return(true);
+ return true;
}
-
-
+
private:
std::istream& fp_;
bool _M_ok;
BedType* _M_value;
std::string fn_;
Bed::CoordType cnt_;
- std::string lastChr_;
+ std::string lastChr_, lastRest_;
Bed::CoordType lastStart_, lastEnd_;
bool allowHeaders_;
bool nestCheck_;
diff --git a/interfaces/general-headers/data/bed/BedCompare.hpp b/interfaces/general-headers/data/bed/BedCompare.hpp
index 045bf78..8963e7c 100644
--- a/interfaces/general-headers/data/bed/BedCompare.hpp
+++ b/interfaces/general-headers/data/bed/BedCompare.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -26,6 +26,8 @@
#include <cstring>
#include <functional>
+#include <limits>
+#include <type_traits>
namespace Bed {
@@ -66,7 +68,7 @@ namespace Bed {
: private GenomicAddressCompare<BedType1, BedType2> {
typedef GenomicAddressCompare<BedType1, BedType2> Base;
bool operator()(BedType1 const* ptr1, BedType2 const* ptr2) const {
- return !Base::operator()(ptr1, ptr2);
+ return Base::operator()(ptr2, ptr1);
}
};
@@ -95,6 +97,103 @@ namespace Bed {
};
template <typename BedType1, typename BedType2 = BedType1>
+ struct CoordRestCompare // not caring about chrom here
+ : public std::binary_function<BedType1 const*, BedType2 const*, bool> {
+
+ template <typename T=BedType1, typename U=BedType2>
+ typename std::enable_if<T::UseRest && U::UseRest, bool>::type
+ operator()(BedType1 const* one, BedType2 const* two) const {
+ if ( one->start() != two->start() )
+ return one->start() < two->start();
+ if ( one->end() != two->end() )
+ return one->end() < two->end();
+ return std::strcmp(one->full_rest(), two->full_rest()) < 0;
+ }
+
+ template <typename T=BedType1, typename U=BedType2>
+ typename std::enable_if<T::UseRest && !U::UseRest, bool>::type
+ operator()(BedType1 const* one, BedType2 const* two) const {
+ if ( one->start() != two->start() )
+ return one->start() < two->start();
+ if ( one->end() != two->end() )
+ return one->end() < two->end();
+ return std::strlen(one->full_rest()) == 0;
+ }
+
+ template <typename T=BedType1, typename U=BedType2>
+ typename std::enable_if<!T::UseRest && U::UseRest, bool>::type
+ operator()(BedType1 const* one, BedType2 const* two) const {
+ if ( one->start() != two->start() )
+ return one->start() < two->start();
+ if ( one->end() != two->end() )
+ return one->end() < two->end();
+ std::strlen(two->full_rest()) != 0;
+ }
+
+ template <typename T=BedType1, typename U=BedType2>
+ typename std::enable_if<!T::UseRest && !U::UseRest, bool>::type
+ operator()(BedType1 const* one, BedType2 const* two) const {
+ if ( one->start() != two->start() )
+ return one->start() < two->start();
+ return one->end() < two->end();
+ }
+ };
+
+ template <typename BedType1, typename BedType2 = BedType1>
+ struct CoordRestAddressCompare // not caring about chrom here
+ : public std::binary_function<BedType1 const*, BedType2 const*, bool> {
+
+ template <typename T=BedType1, typename U=BedType2>
+ typename std::enable_if<T::UseRest && U::UseRest, bool>::type
+ operator()(BedType1 const* one, BedType2 const* two) const {
+ if ( one->start() != two->start() )
+ return one->start() < two->start();
+ if ( one->end() != two->end() )
+ return one->end() < two->end();
+ int val = std::strcmp(one->full_rest(), two->full_rest());
+ if ( val != 0 )
+ return val < 0;
+ return one < two;
+ }
+
+ template <typename T=BedType1, typename U=BedType2>
+ typename std::enable_if<T::UseRest && !U::UseRest, bool>::type
+ operator()(BedType1 const* one, BedType2 const* two) const {
+ if ( one->start() != two->start() )
+ return one->start() < two->start();
+ if ( one->end() != two->end() )
+ return one->end() < two->end();
+ auto n = std::strlen(one->full_rest());
+ if ( n != 0 )
+ return false;
+ return one < two;
+ }
+
+ template <typename T=BedType1, typename U=BedType2>
+ typename std::enable_if<!T::UseRest && U::UseRest, bool>::type
+ operator()(BedType1 const* one, BedType2 const* two) const {
+ if ( one->start() != two->start() )
+ return one->start() < two->start();
+ if ( one->end() != two->end() )
+ return one->end() < two->end();
+ auto n = std::strlen(two->full_rest());
+ if ( n != 0 )
+ return true;
+ return one < two;
+ }
+
+ template <typename T=BedType1, typename U=BedType2>
+ typename std::enable_if<!T::UseRest && !U::UseRest, bool>::type
+ operator()(BedType1 const* one, BedType2 const* two) const {
+ if ( one->start() != two->start() )
+ return one->start() < two->start();
+ if ( one->end() != two->end() )
+ return one->end() < two->end();
+ return one < two;
+ }
+ };
+
+ template <typename BedType1, typename BedType2 = BedType1>
struct RevCoordAddressCompare // not caring about chrom here
: public std::binary_function<BedType1 const*, BedType2 const*, bool> {
@@ -160,6 +259,33 @@ namespace Bed {
}
};
+ template <typename BedType1, typename BedType2 = BedType1>
+ struct ScoreThenGenomicCompareLesser
+ : public std::binary_function<BedType1 const*, BedType2 const*, bool> {
+
+ inline
+ bool operator()(BedType1 const* one, BedType2 const* two) const {
+ if ( one->measurement() != two->measurement() )
+ return one->measurement() < two->measurement();
+
+ static int v = 0;
+ if ( (v = std::strcmp(one->chrom(), two->chrom())) != 0 )
+ return v < 0;
+ if ( one->start() != two->start() )
+ return one->start() < two->start();
+ return one->end() < two->end();
+ }
+ };
+
+ template <typename BedType1, typename BedType2 = BedType1>
+ struct ScoreThenGenomicCompareGreater
+ : private ScoreThenGenomicCompareLesser<BedType1, BedType2> {
+ typedef ScoreThenGenomicCompareLesser<BedType1, BedType2> Base;
+ inline
+ bool operator()(BedType1 const* ptr1, BedType2 const* ptr2) const {
+ return Base::operator()(ptr2, ptr1);
+ }
+ };
} // namespace Bed
diff --git a/interfaces/general-headers/data/bed/BedDistances.hpp b/interfaces/general-headers/data/bed/BedDistances.hpp
index f017f8d..d7bad58 100644
--- a/interfaces/general-headers/data/bed/BedDistances.hpp
+++ b/interfaces/general-headers/data/bed/BedDistances.hpp
@@ -5,7 +5,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/data/bed/BedTypes.hpp b/interfaces/general-headers/data/bed/BedTypes.hpp
index dbda930..9d6fd40 100644
--- a/interfaces/general-headers/data/bed/BedTypes.hpp
+++ b/interfaces/general-headers/data/bed/BedTypes.hpp
@@ -5,7 +5,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -38,7 +38,6 @@ namespace Bed {
typedef BasicCoords<UseNonStaticChrom, UseRest> Bed3Type;
typedef Bed4< Bed3Type, UseRest > Bed4Type;
typedef Bed5< Bed4Type, MeasureType, UseRest > Bed5Type;
- typedef Bed6< Bed5Type, UseRest > Bed6Type;
};
enum { Rest = true, NoRest = false, OneChrom = false, AllChrom = true };
@@ -66,12 +65,6 @@ namespace Bed {
typedef BTAllNoRest::Bed5Type B5NoRest;
typedef BTOneRest::Bed5Type B5OneChromRest;
typedef BTOneNoRest::Bed5Type B5OneChromNoRest;
-
- typedef BTAllRest::Bed6Type B6Rest;
- typedef BTAllNoRest::Bed6Type B6NoRest;
- typedef BTOneRest::Bed6Type B6OneChromRest;
- typedef BTOneNoRest::Bed6Type B6OneChromNoRest;
-
} // namespace Bed
#endif // BEDTYPES_HPP
diff --git a/interfaces/general-headers/data/measurement/AssayMeasurement.hpp b/interfaces/general-headers/data/measurement/AssayMeasurement.hpp
index 72c9196..fda890c 100644
--- a/interfaces/general-headers/data/measurement/AssayMeasurement.hpp
+++ b/interfaces/general-headers/data/measurement/AssayMeasurement.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/data/measurement/NaN.hpp b/interfaces/general-headers/data/measurement/NaN.hpp
index 12f7ac6..8160b1e 100644
--- a/interfaces/general-headers/data/measurement/NaN.hpp
+++ b/interfaces/general-headers/data/measurement/NaN.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/data/measurement/SelectMeasureType.hpp b/interfaces/general-headers/data/measurement/SelectMeasureType.hpp
index 30d2703..73be995 100644
--- a/interfaces/general-headers/data/measurement/SelectMeasureType.hpp
+++ b/interfaces/general-headers/data/measurement/SelectMeasureType.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/data/starch/starchApi.hpp b/interfaces/general-headers/data/starch/starchApi.hpp
index 881922f..536f516 100644
--- a/interfaces/general-headers/data/starch/starchApi.hpp
+++ b/interfaces/general-headers/data/starch/starchApi.hpp
@@ -6,7 +6,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -163,6 +163,7 @@ namespace starch
json_t *_mdJSONStreams = NULL;
char *_archStreamChr = NULL;
char *_archStreamFn = NULL;
+ char *_archStreamSignature = NULL;
size_t _mdJSONStreamIdx;
json_t *_mdJSONStream = NULL;
json_t *_mdJSONStreamChr = NULL;
@@ -173,12 +174,14 @@ namespace starch
json_t *_mdJSONStreamUniqueBaseCount = NULL;
json_t *_mdJSONStreamDuplicateElementExists = NULL;
json_t *_mdJSONStreamNestedElementExists = NULL;
+ // json_t *_mdJSONStreamSignature = NULL;
uint64_t _archStreamSize = 0;
Bed::LineCountType _archStreamLineCount = 0;
Bed::BaseCountType _archStreamNonUniqueBaseCount = 0;
Bed::BaseCountType _archStreamUniqueBaseCount = 0;
Boolean _archStreamDuplicateElementExists = STARCH_DEFAULT_DUPLICATE_ELEMENT_FLAG_VALUE;
Boolean _archStreamNestedElementExists = STARCH_DEFAULT_NESTED_ELEMENT_FLAG_VALUE;
+ LineLengthType _archStreamMaxElementLength = STARCH_DEFAULT_LINE_STRING_LENGTH;
Metadata *_testMd = NULL;
Metadata *_firstMd = NULL;
unsigned int _recIdx = 0U;
@@ -399,7 +402,7 @@ namespace starch
}
else
_archStreamNestedElementExists = static_cast<Boolean>( json_is_true(_mdJSONStreamNestedElementExists) );
-
+ // we skip _mdJSONStreamSignature in v1 metadata
if (_recIdx == 0) {
_testMd = STARCH_createMetadata(_archStreamChr,
_archStreamFn,
@@ -408,7 +411,9 @@ namespace starch
_archStreamNonUniqueBaseCount,
_archStreamUniqueBaseCount,
_archStreamDuplicateElementExists,
- _archStreamNestedElementExists);
+ _archStreamNestedElementExists,
+ _archStreamSignature,
+ _archStreamMaxElementLength);
_firstMd = _testMd;
}
else
@@ -420,7 +425,9 @@ namespace starch
_archStreamNonUniqueBaseCount,
_archStreamUniqueBaseCount,
_archStreamDuplicateElementExists,
- _archStreamNestedElementExists);
+ _archStreamNestedElementExists,
+ _archStreamSignature,
+ _archStreamMaxElementLength);
_recIdx++;
_mdJSONStreamChr = NULL;
_mdJSONStreamFn = NULL;
@@ -502,6 +509,7 @@ namespace starch
Bed::BaseCountType _recUniqueBaseCountValue = 0;
Boolean _recDuplicateElementExists = STARCH_DEFAULT_DUPLICATE_ELEMENT_FLAG_VALUE;
Boolean _recNestedElementExists = STARCH_DEFAULT_NESTED_ELEMENT_FLAG_VALUE;
+ LineLengthType _recStreamMaxElementLength = STARCH_DEFAULT_LINE_STRING_LENGTH;
/* read first 8 kilobytes into buffer */
@@ -570,7 +578,9 @@ namespace starch
_recNonUniqueBaseCountValue,
_recUniqueBaseCountValue,
_recDuplicateElementExists,
- _recNestedElementExists);
+ _recNestedElementExists,
+ NULL,
+ _recStreamMaxElementLength);
_firstMd = _testMd;
}
else
@@ -582,7 +592,9 @@ namespace starch
_recNonUniqueBaseCountValue,
_recUniqueBaseCountValue,
_recDuplicateElementExists,
- _recNestedElementExists);
+ _recNestedElementExists,
+ NULL,
+ _recStreamMaxElementLength);
}
else
break;
@@ -1620,6 +1632,14 @@ namespace starch
#endif
//if (zError != Z_STREAM_END) {
//if (zHave > 0) {
+
+ if (needToReadZChunk && (zError != Z_STREAM_END)) {
+#ifdef DEBUG
+ std::fprintf(stderr, "--> needed to read a new chunk because we're in the middle of an incomplete line\n");
+#endif
+ zReadLine();
+ }
+
if (zHave >= zOutBufIdx && !postBreakdownZValuesIdentical) {
#ifdef DEBUG
std::fprintf(stderr, "--> PRE - zOutBufIdx [ %d ] zHave [ %d ] postBreakdownZValuesIdentical [ %d ]\n", zOutBufIdx, zHave, postBreakdownZValuesIdentical);
@@ -1857,16 +1877,21 @@ namespace starch
}
zHave = (STARCH_Z_CHUNK * STARCH_Z_CHUNK_MULTIPLIER) - static_cast<int>( zStream.avail_out );
zOutBuf[zHave] = '\0';
+ zOutBufIdx = 0;
/* copy remainder buffer onto line buffer, if not NULL */
if (zRemainderBuf) {
- strncpy(reinterpret_cast<char *>( zLineBuf ), reinterpret_cast<const char *>( zRemainderBuf ), strlen(reinterpret_cast<const char *>( zRemainderBuf )));
+ strncpy(reinterpret_cast<char *>( zLineBuf ), reinterpret_cast<const char *>( zRemainderBuf ), strlen(reinterpret_cast<const char *>( zRemainderBuf )) + 1);
zBufOffset = strlen(reinterpret_cast<const char *>( zRemainderBuf ));
}
else {
zBufOffset = 0;
}
#ifdef DEBUG
- std::fprintf(stderr, "--> zOutBuf [ %s ]\n", zOutBuf);
+ std::fprintf(stderr, "--> zOutBuf [ %s ]\n", zOutBuf);
+ std::fprintf(stderr, "--> zLineBuf [ %s ]\n", zLineBuf);
+ std::fprintf(stderr, "--> zOutBufIdx [ %d ]\n", zOutBufIdx);
+ std::fprintf(stderr, "--> zBufOffset [ %d ]\n", zBufOffset);
+ std::fprintf(stderr, "--> zBufIdx [ %d ]\n", zBufIdx);
#endif
needToInflateZChunk = false;
}
@@ -1881,9 +1906,18 @@ namespace starch
#endif
return EXIT_SUCCESS;
}
- else
+ else {
zLineBuf[zBufIdx] = zOutBuf[zOutBufIdx - 1];
+ zLineBuf[zBufIdx + 1] = '\0';
+#ifdef DEBUG
+ std::fprintf(stderr, "--> incomplete [ %s ]\n", zLineBuf);
+#endif
+ }
}
+ zLineBuf[zBufIdx] = '\0';
+#ifdef DEBUG
+ std::fprintf(stderr, "--> zLineBuf (POST) [ %s ]\n", zLineBuf);
+#endif
if (strlen(reinterpret_cast<const char *>( zLineBuf )) > 0) {
if (strlen(reinterpret_cast<const char *>( zLineBuf )) > strlen(reinterpret_cast<const char *>( zRemainderBuf ))) {
@@ -1901,6 +1935,9 @@ namespace starch
strncpy(reinterpret_cast<char *>( zRemainderBuf ), reinterpret_cast<const char *>( zLineBuf ), zBufIdx);
zRemainderBuf[zBufIdx] = '\0';
+#ifdef DEBUG
+ fprintf(stderr, "--> remainder set to: [ %s ]\n", zRemainderBuf);
+#endif
/* we should only at most have to do this every STARCH_Z_CHUNK chars, and once */
/* at the tail end of a chromosome's worth of a z-stream */
}
diff --git a/interfaces/general-headers/data/starch/starchBase64Coding.h b/interfaces/general-headers/data/starch/starchBase64Coding.h
index 325c0fb..506966e 100644
--- a/interfaces/general-headers/data/starch/starchBase64Coding.h
+++ b/interfaces/general-headers/data/starch/starchBase64Coding.h
@@ -6,7 +6,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/data/starch/starchConstants.h b/interfaces/general-headers/data/starch/starchConstants.h
index 1aa1ade..8b86a21 100644
--- a/interfaces/general-headers/data/starch/starchConstants.h
+++ b/interfaces/general-headers/data/starch/starchConstants.h
@@ -6,7 +6,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/data/starch/starchFileHelpers.h b/interfaces/general-headers/data/starch/starchFileHelpers.h
index 7efca37..a86ee81 100644
--- a/interfaces/general-headers/data/starch/starchFileHelpers.h
+++ b/interfaces/general-headers/data/starch/starchFileHelpers.h
@@ -6,7 +6,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -28,24 +28,6 @@
#define STARCH_FATAL_ERROR -1
-/*
- On Darwin, file I/O is 64-bit by default (OS X 10.5 at least) so we use standard
- types and calls
-
- Via Terry Lambert <tlambert at apple.com>:
-
- ``off_t is 64 bit by default in Mac OS X, so it's not necessary to use special
- calls to obtain large file support. You will still need to use fseeko()
- instead of fseek(), unless you compile your code 64 bit (fseek() takes a long
- instead of an off_t because the standard requires it).''
-*/
-
-#ifdef __APPLE__
-#define fopen64 fopen
-#define off64_t off_t
-#define fseeko64 fseeko
-#endif
-
/*
In Cygwin, there is no support for fseeko(). We can use fseek() on a 64-bit
distribution of Cygwin, because a signed long int has sufficient bits to store
diff --git a/interfaces/general-headers/data/starch/starchHelpers.h b/interfaces/general-headers/data/starch/starchHelpers.h
index b0b73b1..8aebd03 100644
--- a/interfaces/general-headers/data/starch/starchHelpers.h
+++ b/interfaces/general-headers/data/starch/starchHelpers.h
@@ -6,7 +6,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -116,19 +116,28 @@ int STARCH2_transformInput(unsigned char **header,
const CompressionType compressionType,
const char *tag,
const char *note,
- const Boolean headerFlag);
+ const Boolean generatePerChrSignatureFlag,
+ const Boolean headerFlag,
+ const Boolean reportProgressFlag,
+ const LineCountType reportProgressN);
int STARCH2_transformHeaderedBEDInput(const FILE *inFp,
Metadata **md,
const CompressionType compressionType,
const char *tag,
- const char *note);
+ const char *note,
+ const Boolean generatePerChrSignatureFlag,
+ const Boolean reportProgressFlag,
+ const LineCountType reportProgressN);
int STARCH2_transformHeaderlessBEDInput(const FILE *inFp,
Metadata **md,
const CompressionType compressionType,
const char *tag,
- const char *note);
+ const char *note,
+ const Boolean generatePerChrSignatureFlag,
+ const Boolean reportProgressFlag,
+ const LineCountType reportProgressN);
int STARCH2_writeStarchHeaderToOutputFp(const unsigned char *header,
const FILE *fp);
diff --git a/interfaces/general-headers/data/starch/starchMetadataHelpers.h b/interfaces/general-headers/data/starch/starchMetadataHelpers.h
index fbe2595..f239572 100644
--- a/interfaces/general-headers/data/starch/starchMetadataHelpers.h
+++ b/interfaces/general-headers/data/starch/starchMetadataHelpers.h
@@ -6,7 +6,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -37,19 +37,13 @@
#include "jansson.h"
-/* current "stable" binary version: 2.4.2 */
-/* current "dev" binary version: 2.5.0 */
-
-/* current "stable" archive version: 2.0.0 */
-/* current "dev" archive version: 2.1.0 */
-
#ifdef __cplusplus
namespace starch {
using namespace Bed;
#endif
#define STARCH_MAJOR_VERSION 2
-#define STARCH_MINOR_VERSION 1
+#define STARCH_MINOR_VERSION 2
#define STARCH_REVISION_VERSION 0
#define STARCH_DEFAULT_COMPRESSION_TYPE kBzip2
@@ -68,6 +62,7 @@
#define STARCH_STREAM_METADATA_FILENAME_MAX_LENGTH 1024
#define STARCH_STREAM_METADATA_MAX_LENGTH 1048576
#define STARCH_DEFAULT_LINE_COUNT 0
+#define STARCH_DEFAULT_LINE_STRING_LENGTH 0
#define STARCH_DEFAULT_NON_UNIQUE_BASE_COUNT 0
#define STARCH_DEFAULT_UNIQUE_BASE_COUNT 0
#ifdef __cplusplus
@@ -85,8 +80,10 @@
#define STARCH_METADATA_STREAM_LIST_KEY "streams"
#define STARCH_METADATA_STREAM_CHROMOSOME_KEY "chromosome"
#define STARCH_METADATA_STREAM_FILENAME_KEY "filename"
+#define STARCH_METADATA_STREAM_SIGNATURE_KEY "signature"
#define STARCH_METADATA_STREAM_SIZE_KEY "size"
#define STARCH_METADATA_STREAM_LINECOUNT_KEY "uncompressedLineCount"
+#define STARCH_METADATA_STREAM_LINEMAXSTRINGLENGTH_KEY "uncompressedLineMaxStringLength"
#define STARCH_METADATA_STREAM_TOTALNONUNIQUEBASES_KEY "nonUniqueBaseCount"
#define STARCH_METADATA_STREAM_TOTALUNIQUEBASES_KEY "uniqueBaseCount"
#define STARCH_METADATA_STREAM_DUPLICATEELEMENTEXISTS_KEY "duplicateElementExists"
@@ -156,10 +153,12 @@ typedef struct metadata {
char *filename;
uint64_t size;
LineCountType lineCount;
+ LineLengthType lineMaxStringLength;
BaseCountType totalNonUniqueBases;
BaseCountType totalUniqueBases;
Boolean duplicateElementExists;
Boolean nestedElementExists;
+ char *signature;
struct metadata *next;
} Metadata;
@@ -188,16 +187,6 @@ typedef enum {
typedef
unsigned int HeaderFlag;
-/*
- On Darwin, file I/O is 64-bit by default (OS X 10.5 at least) so we use standard
- types and calls
-*/
-
-#ifdef __APPLE__
-#define off64_t off_t
-#define fopen64 fopen
-#endif
-
Metadata * STARCH_createMetadata(char const *chr,
char const *fn,
uint64_t size,
@@ -205,7 +194,9 @@ Metadata * STARCH_createMetadata(char const *chr,
BaseCountType totalNonUniqueBases,
BaseCountType totalUniqueBases,
Boolean duplicateElementExists,
- Boolean nestedElementExists);
+ Boolean nestedElementExists,
+ char const *signature,
+ LineLengthType lineMaxStringLength);
Metadata * STARCH_addMetadata(Metadata *md,
char *chr,
@@ -215,7 +206,9 @@ Metadata * STARCH_addMetadata(Metadata *md,
BaseCountType totalNonUniqueBases,
BaseCountType totalUniqueBases,
Boolean duplicateElementExists,
- Boolean nestedElementExists);
+ Boolean nestedElementExists,
+ char *signature,
+ LineLengthType lineMaxStringLength);
Metadata * STARCH_copyMetadata(const Metadata *md);
@@ -227,7 +220,9 @@ int STARCH_updateMetadataForChromosome(Metadata **md,
BaseCountType totalNonUniqueBases,
BaseCountType totalUniqueBases,
Boolean duplicateElementExists,
- Boolean nestedElementExists);
+ Boolean nestedElementExists,
+ char *signature,
+ LineLengthType lineMaxStringLength);
int STARCH_listMetadata(const Metadata *md,
const char *chr);
@@ -299,6 +294,9 @@ ArchiveVersion * STARCH_copyArchiveVersion(const ArchiveVersion *oav);
int STARCH_chromosomeInMetadataRecords(const Metadata *md,
const char *chr);
+int STARCH_chromosomePositionedBeforeExistingMetadataRecord(const Metadata *md,
+ const char *chr);
+
#ifdef __cplusplus
} // namespace starch
#endif
diff --git a/interfaces/general-headers/data/starch/starchSha1Digest.h b/interfaces/general-headers/data/starch/starchSha1Digest.h
index 10aeb06..2cce820 100644
--- a/interfaces/general-headers/data/starch/starchSha1Digest.h
+++ b/interfaces/general-headers/data/starch/starchSha1Digest.h
@@ -100,15 +100,13 @@ extern void sha1_init_ctx (struct sha1_ctx *ctx);
initialization function update the context for the next LEN bytes
starting at BUFFER.
It is necessary that LEN is a multiple of 64!!! */
-extern void sha1_process_block (const void *buffer, size_t len,
- struct sha1_ctx *ctx);
+extern void sha1_process_block (const void *buffer, size_t len, struct sha1_ctx *ctx);
/* Starting with the result of former calls of this function (or the
initialization function update the context for the next LEN bytes
starting at BUFFER.
It is NOT required that LEN is a multiple of 64. */
-extern void sha1_process_bytes (const void *buffer, size_t len,
- struct sha1_ctx *ctx);
+extern void sha1_process_bytes (const void *buffer, size_t len, struct sha1_ctx *ctx);
/* Process the remaining bytes in the buffer and put result from CTX
in first 20 bytes following RESBUF. The result is always in little
diff --git a/interfaces/general-headers/data/starch/unstarchHelpers.h b/interfaces/general-headers/data/starch/unstarchHelpers.h
index 97222d2..8c2b0be 100644
--- a/interfaces/general-headers/data/starch/unstarchHelpers.h
+++ b/interfaces/general-headers/data/starch/unstarchHelpers.h
@@ -6,7 +6,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -69,6 +69,10 @@
#define UNSTARCH_ELEMENT_NESTED_CHR_STR_ERROR 30
#define UNSTARCH_ELEMENT_NESTED_ALL_STR_ERROR 31
#define UNSTARCH_IS_STARCH_ARCHIVE_ERROR 32
+#define UNSTARCH_SIGNATURE_ERROR 33
+#define UNSTARCH_SIGNATURE_VERIFY_ERROR 34
+#define UNSTARCH_ELEMENT_MAX_STRING_LENGTH_CHR_ERROR 35
+#define UNSTARCH_ELEMENT_MAX_STRING_LENGTH_ALL_ERROR 36
int UNSTARCH_reverseTransformInput(const char *chr,
const unsigned char *str,
@@ -189,24 +193,32 @@ char * UNSTARCH_strndup(const char *s,
void UNSTARCH_bzReadLine(BZFILE *input,
unsigned char **output);
-LineCountType UNSTARCH_lineCountForChromosome(const Metadata *md,
- const char *chr);
+LineCountType UNSTARCH_lineCountForChromosome(const Metadata *md,
+ const char *chr);
void UNSTARCH_printLineCountForChromosome(const Metadata *md,
const char *chr);
void UNSTARCH_printLineCountForAllChromosomes(const Metadata *md);
-BaseCountType UNSTARCH_nonUniqueBaseCountForChromosome(const Metadata *md,
- const char *chr);
+LineLengthType UNSTARCH_lineMaxStringLengthForChromosome(const Metadata *md,
+ const char *chr);
+
+void UNSTARCH_printLineMaxStringLengthForChromosome(const Metadata *md,
+ const char *chr);
+
+void UNSTARCH_printLineMaxStringLengthForAllChromosomes(const Metadata *md);
+
+BaseCountType UNSTARCH_nonUniqueBaseCountForChromosome(const Metadata *md,
+ const char *chr);
void UNSTARCH_printNonUniqueBaseCountForChromosome(const Metadata *md,
const char *chr);
void UNSTARCH_printNonUniqueBaseCountForAllChromosomes(const Metadata *md);
-BaseCountType UNSTARCH_uniqueBaseCountForChromosome(const Metadata *md,
- const char *chr);
+BaseCountType UNSTARCH_uniqueBaseCountForChromosome(const Metadata *md,
+ const char *chr);
void UNSTARCH_printUniqueBaseCountForChromosome(const Metadata *md,
const char *chr);
@@ -239,6 +251,37 @@ void UNSTARCH_printNestedElementExistsIntegerForChromosome(const M
void UNSTARCH_printNestedElementExistsIntegersForAllChromosomes(const Metadata *md);
+void UNSTARCH_printSignature(const Metadata *md,
+ const char *chr,
+ const unsigned char *mdSha1Buffer);
+
+char * UNSTARCH_signatureForChromosome(const Metadata *md,
+ const char *chr);
+
+void UNSTARCH_printAllSignatures(const Metadata *md,
+ const unsigned char *mdSha1Buffer);
+
+void UNSTARCH_printMetadataSignature(const unsigned char *mdSha1Buffer);
+
+Boolean UNSTARCH_verifySignature(FILE **inFp,
+ const Metadata *md,
+ const uint64_t mdOffset,
+ const char *chr,
+ CompressionType compType);
+
+char * UNSTARCH_observedSignatureForChromosome(FILE **inFp,
+ const Metadata *md,
+ const uint64_t mdOffset,
+ const char *chr,
+ CompressionType compType);
+
+Boolean UNSTARCH_verifyAllSignatures(FILE **inFp,
+ const Metadata *md,
+ const uint64_t mdOffset,
+ CompressionType compType);
+
+void UNSTARCH_printAllChromosomeSignatures(const Metadata *md);
+
const char * UNSTARCH_booleanToString(const Boolean val);
int UNSTARCH_reverseTransformCoordinates(const LineCountType lineIdx,
diff --git a/interfaces/general-headers/suite/BEDOPS.Constants.hpp b/interfaces/general-headers/suite/BEDOPS.Constants.hpp
index c3fcb84..6dc7efd 100644
--- a/interfaces/general-headers/suite/BEDOPS.Constants.hpp
+++ b/interfaces/general-headers/suite/BEDOPS.Constants.hpp
@@ -6,7 +6,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -39,30 +39,34 @@
#define INT_TOKEN_CHR_FIELD_INDEX 0
#define INT_TOKEN_START_FIELD_INDEX 1
#define INT_TOKEN_STOP_FIELD_INDEX 2
-
+#define INT_MEM_CHUNK_SZ 64 // how many BED elements allocated at a time
#ifdef __cplusplus
#include <cinttypes>
+#include <cstddef>
+
namespace Bed {
// Use these typedef's in applications
typedef uint64_t CoordType;
typedef int64_t SignedCoordType;
typedef CoordType LineCountType;
typedef CoordType BaseCountType;
+ typedef int LineLengthType;
static_assert(sizeof(SignedCoordType) >= sizeof(INT_MAX_COORD_VALUE), "INT_MAX_COORD_VALUE is too big!"); // expected-warning {{static_assert declarations are incompatible with C++98}}
- const unsigned long TOKEN_CHR_MAX_LENGTH = INT_TOKEN_CHR_MAX_LENGTH;
- const unsigned long TOKEN_ID_MAX_LENGTH = INT_TOKEN_ID_MAX_LENGTH;
- const unsigned long TOKEN_REST_MAX_LENGTH = INT_TOKEN_REST_MAX_LENGTH;
- const unsigned long MAX_DEC_INTEGERS = INT_MAX_DEC_INTEGERS;
- const SignedCoordType MAX_COORD_VALUE = INT_MAX_COORD_VALUE;
- const unsigned long TOKENS_MAX_LENGTH = INT_TOKENS_MAX_LENGTH;
- const unsigned long TOKENS_HEADER_MAX_LENGTH = INT_TOKENS_HEADER_MAX_LENGTH;
- const unsigned long TOKEN_CHR_FIELD_INDEX = INT_TOKEN_CHR_FIELD_INDEX;
- const unsigned long TOKEN_START_FIELD_INDEX = INT_TOKEN_START_FIELD_INDEX;
- const unsigned long TOKEN_STOP_FIELD_INDEX = INT_TOKEN_STOP_FIELD_INDEX;
+ constexpr LineLengthType TOKEN_CHR_MAX_LENGTH = INT_TOKEN_CHR_MAX_LENGTH;
+ constexpr LineLengthType TOKEN_ID_MAX_LENGTH = INT_TOKEN_ID_MAX_LENGTH;
+ constexpr LineLengthType TOKEN_REST_MAX_LENGTH = INT_TOKEN_REST_MAX_LENGTH;
+ constexpr unsigned long MAX_DEC_INTEGERS = INT_MAX_DEC_INTEGERS;
+ constexpr SignedCoordType MAX_COORD_VALUE = INT_MAX_COORD_VALUE;
+ constexpr LineLengthType TOKENS_MAX_LENGTH = INT_TOKENS_MAX_LENGTH;
+ constexpr LineLengthType TOKENS_HEADER_MAX_LENGTH = INT_TOKENS_HEADER_MAX_LENGTH;
+ constexpr unsigned long TOKEN_CHR_FIELD_INDEX = INT_TOKEN_CHR_FIELD_INDEX;
+ constexpr unsigned long TOKEN_START_FIELD_INDEX = INT_TOKEN_START_FIELD_INDEX;
+ constexpr unsigned long TOKEN_STOP_FIELD_INDEX = INT_TOKEN_STOP_FIELD_INDEX;
+ constexpr std::size_t CHUNKSZ = INT_MEM_CHUNK_SZ;
} // namespace Bed
#else
@@ -74,6 +78,7 @@ namespace Bed {
typedef int64_t SignedCoordType;
typedef CoordType LineCountType;
typedef CoordType BaseCountType;
+ typedef int LineLengthType;
#define TOKEN_CHR_MAX_LENGTH INT_TOKEN_CHR_MAX_LENGTH
#define TOKEN_ID_MAX_LENGTH INT_TOKEN_ID_MAX_LENGTH
@@ -85,6 +90,7 @@ namespace Bed {
#define TOKEN_CHR_FIELD_INDEX INT_TOKEN_CHR_FIELD_INDEX
#define TOKEN_START_FIELD_INDEX INT_TOKEN_START_FIELD_INDEX
#define TOKEN_STOP_FIELD_INDEX INT_TOKEN_STOP_FIELD_INDEX
+#define CHUNKSZ INT_MEM_CHUNK_SZ
#endif
diff --git a/interfaces/general-headers/suite/BEDOPS.Version.hpp b/interfaces/general-headers/suite/BEDOPS.Version.hpp
index 183f16c..b61da2b 100644
--- a/interfaces/general-headers/suite/BEDOPS.Version.hpp
+++ b/interfaces/general-headers/suite/BEDOPS.Version.hpp
@@ -8,7 +8,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -34,7 +34,7 @@ namespace BEDOPS {
#endif
static const char* revision() {
- return("2.4.20");
+ return("2.4.26");
}
static const char* citation() {
diff --git a/interfaces/general-headers/utility/AllocateIterator.hpp b/interfaces/general-headers/utility/AllocateIterator.hpp
index 27a257f..96e566a 100644
--- a/interfaces/general-headers/utility/AllocateIterator.hpp
+++ b/interfaces/general-headers/utility/AllocateIterator.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/utility/Assertion.hpp b/interfaces/general-headers/utility/Assertion.hpp
index 979ed05..f153371 100755
--- a/interfaces/general-headers/utility/Assertion.hpp
+++ b/interfaces/general-headers/utility/Assertion.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/utility/ByLine.hpp b/interfaces/general-headers/utility/ByLine.hpp
index ec0b65f..8c1f2d9 100755
--- a/interfaces/general-headers/utility/ByLine.hpp
+++ b/interfaces/general-headers/utility/ByLine.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/utility/Exception.hpp b/interfaces/general-headers/utility/Exception.hpp
index 020ae16..7a01085 100755
--- a/interfaces/general-headers/utility/Exception.hpp
+++ b/interfaces/general-headers/utility/Exception.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/utility/FPWrap.hpp b/interfaces/general-headers/utility/FPWrap.hpp
index b431110..a070fcd 100644
--- a/interfaces/general-headers/utility/FPWrap.hpp
+++ b/interfaces/general-headers/utility/FPWrap.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/utility/Factory.hpp b/interfaces/general-headers/utility/Factory.hpp
index 3fa0a33..8e108b6 100644
--- a/interfaces/general-headers/utility/Factory.hpp
+++ b/interfaces/general-headers/utility/Factory.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/utility/Formats.hpp b/interfaces/general-headers/utility/Formats.hpp
index 3149d06..ba82779 100644
--- a/interfaces/general-headers/utility/Formats.hpp
+++ b/interfaces/general-headers/utility/Formats.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/utility/IteratorPair.hpp b/interfaces/general-headers/utility/IteratorPair.hpp
index 4102a8e..2441374 100644
--- a/interfaces/general-headers/utility/IteratorPair.hpp
+++ b/interfaces/general-headers/utility/IteratorPair.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/utility/OrderCompare.hpp b/interfaces/general-headers/utility/OrderCompare.hpp
index 798cf77..8db653e 100644
--- a/interfaces/general-headers/utility/OrderCompare.hpp
+++ b/interfaces/general-headers/utility/OrderCompare.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/utility/PooledMemory.hpp b/interfaces/general-headers/utility/PooledMemory.hpp
new file mode 100644
index 0000000..d094b55
--- /dev/null
+++ b/interfaces/general-headers/utility/PooledMemory.hpp
@@ -0,0 +1,508 @@
+/*
+ Author: Shane Neph
+ Date: Tue Nov 22 13:53:28 PST 2016
+*/
+//
+// BEDOPS
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+
+
+#ifndef __UTILS_SIMPLE_SMALLOBJ_MEM__
+#define __UTILS_SIMPLE_SMALLOBJ_MEM__
+
+#include <algorithm>
+#include <array>
+#include <bitset>
+#include <limits>
+#include <map>
+#include <ostream>
+#include <set>
+#include <type_traits>
+#include <utility>
+
+namespace Ext {
+
+ template <std::size_t Base, std::size_t L, bool b=(L<=Base)>
+ struct IntLogN {
+ static constexpr std::size_t value = 1 + IntLogN<Base, L/Base + (L%Base > 0)>::value;
+ };
+
+ template <std::size_t Base, std::size_t L>
+ struct IntLogN<Base, L, true> {
+ static constexpr std::size_t value = 1;
+ };
+
+ template <std::size_t V, std::size_t W>
+ struct Pow {
+ static constexpr std::size_t value = V*Pow<V, W-1>::value;
+ };
+
+ template <std::size_t V>
+ struct Pow<V,1> {
+ static constexpr std::size_t value = V;
+ };
+
+ template <std::size_t V>
+ struct Pow<V,0> {
+ static constexpr std::size_t value = 0;
+ };
+
+ template <std::size_t Base,
+ std::size_t Total, // undefined when Check==false
+ bool Check=(Base==std::numeric_limits<unsigned char>::digits) &&
+ (Total%Base==0) &&
+ (Pow<Base, IntLogN<Base, Total>::value>::value==Total)>
+ struct BitMonitor2;
+
+ template <std::size_t Base,
+ std::size_t Total>
+ struct BitMonitor2<Base, Total, true> {
+ static constexpr std::size_t BASE = std::numeric_limits<unsigned char>::digits;
+ static constexpr std::size_t TOTAL = Total + BitMonitor2<Base, Total/Base>::TOTAL;
+ static constexpr std::size_t BITS = Total;
+ static constexpr std::size_t npos = std::numeric_limits<std::size_t>::max();
+
+ BitMonitor2() : _nxt(0) { for( std::size_t i=0; i<TBYTE; ++i) _charset[i]=CZERO; }
+ inline bool any() const { return _charset[LAST_BYTE] != FULL; }
+ inline std::size_t get_open() const { return _nxt; }
+ inline std::size_t size() const { return BITS; }
+
+ inline bool set(std::size_t bit) {
+ std::size_t bin = bit/BASE;
+ _charset[bin] |= (1 << (bit%BASE));
+ if ( _charset[bin] != FULL ) {
+ if ( bit == _nxt ) {
+ for ( std::size_t i = 0; i < BASE; ++i ) {
+ if ( !(_charset[bin] & (1 << i)) ) {
+ _nxt = BASE*bin+i;
+ break;
+ }
+ } // for
+ }
+ return true;
+ }
+
+ std::size_t idx = BITS;
+ std::size_t val = bin; // val == bit/BASE
+ bin = BITS;
+ std::size_t parent = (bin + val);
+ std::size_t pbyte = parent/BASE;
+ while ( parent < TOTAL ) {
+ _charset[pbyte] |= (1 << (val%BASE));
+ if ( _charset[pbyte] != FULL ) {
+ if ( _nxt != bit )
+ return true;
+ _nxt = npos;
+ for ( std::size_t i = 0; i < BASE; ++i ) {
+ if ( !(_charset[pbyte] & (1 << i)) ) {
+ _nxt = find_open_child(pbyte*BASE+i);
+ break;
+ }
+ } // for
+ return true;
+ }
+ idx /= BASE;
+ bin += idx;
+ val /= BASE;
+ parent = bin + val;
+ pbyte = parent/BASE;
+ } // while
+ return false;
+ }
+
+ inline void unset(std::size_t bit) {
+ std::size_t bin = bit/BASE;
+ bool pstatus = _charset[bin] != FULL;
+ _charset[bin] &= ~(1 << (bit%BASE));
+ _nxt = bit;
+ if ( pstatus ) // parent wasn't FULL even before unsetting bit
+ return;
+
+ // unset parent from FULL
+ std::size_t parent_bit = BITS + bin; // bin=bit/BASE
+ _charset[parent_bit/BASE] &= ~(1 << (parent_bit%BASE));
+
+ std::size_t val = BITS/BASE;
+ std::size_t idx = BITS + val;
+ bin /= BASE;
+ while ( !pstatus && (parent_bit = idx + bin) < TOTAL ) {
+ if ( _charset[parent_bit/BASE] != FULL )
+ pstatus = true;
+ _charset[parent_bit/BASE] &= ~(1 << (parent_bit%BASE));
+ val /= BASE;
+ idx += val;
+ bin /= BASE;
+ } // while
+ }
+
+ private:
+ inline std::size_t find_open_child(std::size_t p) {
+ while ( BITS <= p ) {
+ std::size_t isum = BITS;
+ std::size_t inow = BITS;
+ std::size_t itrail = 0, ptrail = 0;
+ while ( isum < p ) {
+ inow /= BASE;
+ ptrail = itrail;
+ itrail = isum;
+ isum += inow;
+ } // while
+ if ( isum > p ) { isum = itrail; itrail = ptrail; }
+ p -= isum;
+ p *= BASE;
+ p += itrail;
+ std::size_t pbyte = p/BASE;
+ for ( std::size_t i = 0; i < BASE; ++i ) {
+ if ( !(_charset[pbyte] & (1 << i)) ) {
+ p += i;
+ break;
+ }
+ } // for
+ } // while
+ return p;
+ }
+
+ template <std::size_t A, std::size_t B, bool C>
+ friend
+ typename std::enable_if<C, std::ostream&>::type
+ operator<<(std::ostream& os, const BitMonitor2<A,B,C>& r);
+
+ private:
+ static constexpr unsigned char FULL = std::numeric_limits<unsigned char>::max();
+ static constexpr unsigned char CZERO = (unsigned char)0;
+ static constexpr std::size_t TBYTE = TOTAL/BASE;
+ static constexpr std::size_t LAST_BYTE = TBYTE-1;
+ std::size_t _nxt;
+ std::array<unsigned char, TBYTE> _charset;
+ };
+
+ template <std::size_t Base>
+ struct BitMonitor2<Base, Base, true> {
+ static constexpr std::size_t BASE = Base;
+ static constexpr std::size_t TOTAL = Base;
+ };
+
+ template <std::size_t A, std::size_t B, bool C>
+ typename std::enable_if<C, std::ostream&>::type
+ operator<<(std::ostream& os, const BitMonitor2<A,B,C>& r) {
+ std::copy(r._charset.begin(), r._charset.end(), std::ostream_iterator<std::bitset<8>>(os, " "));
+ os << std::endl;
+ return os;
+ }
+
+ namespace Zeros {
+ std::size_t NC_ZERO = 0;
+ static constexpr std::size_t C_ZERO = 0;
+ } // namespace Zeros
+
+ /* forcing N to be a power of 8 as less than 8 with a bitset is not that great,
+ though it can be done by modifying Iter requirements.
+ */
+ template <std::size_t N,
+ std::size_t StopLevel,
+ std::size_t Iter=(N%std::numeric_limits<unsigned char>::digits==0) &&
+ (StopLevel%N==0) &&
+ (Pow<N, IntLogN<N, StopLevel>::value>::value==StopLevel)>
+ struct BitMonitor {
+ static constexpr std::size_t NVAL = N;
+ static constexpr std::size_t MYSZ = Pow<NVAL,Iter>::value;
+ static constexpr std::size_t PARENTSZ = Pow<NVAL,Iter-1>::value;
+ static constexpr std::size_t TOTALSZ = MYSZ+PARENTSZ;
+ static constexpr std::size_t LEVEL = Iter;
+ static constexpr bool STOP = false;
+ static constexpr std::size_t npos = std::numeric_limits<std::size_t>::max();
+
+ BitMonitor() : _nxt(0) {}
+
+ inline bool any() const { return _nxt != npos; }
+ inline std::size_t get_open() {
+ // assume any() is true, but perhaps not _children.any()
+ if ( !_children.any() )
+ _children.find_open(_nxt); // ignore rtn value
+ return _children.get_open();
+ }
+
+ std::size_t find_open(std::size_t pbin) {
+ // don't call this - use get_open()
+ pbin *= NVAL;
+ for ( std::size_t idx = pbin; idx < pbin+NVAL; ++idx ) {
+ if ( !_bset[idx] ) {
+ _nxt = idx;
+ return _nxt;
+ }
+ } // for
+ return npos; // something bad just happened
+ }
+
+ // calling program can simply check return value here instead of any()
+ inline bool set(std::size_t bit, std::size_t& cnt=Zeros::NC_ZERO) {
+ std::size_t val = 0;
+ if ( !_children.set(bit, val) ) {
+ _bset[val] = true;
+ cnt = val/NVAL;
+ std::size_t pbin = cnt*NVAL;
+ _nxt = npos;
+ for ( std::size_t idx = pbin; idx < pbin+NVAL; ++idx ) {
+ if ( !_bset[idx] ) {
+ _nxt = idx;
+ break;
+ }
+ } // for
+ return _nxt != npos;
+ }
+ return true;
+ }
+
+ inline void unset(std::size_t bit, std::size_t& cnt=Zeros::NC_ZERO) {
+ _children.unset(bit, cnt);
+ _bset[cnt] = false;
+ _nxt = cnt;
+ cnt /= NVAL;
+ }
+
+ inline std::size_t size() const { return _children.size(); }
+
+ template <std::size_t A, std::size_t B, std::size_t C>
+ friend
+ typename std::enable_if<!BitMonitor<A,B,C>::STOP, std::ostream&>::type
+ operator<<(std::ostream& os, const BitMonitor<A,B,C>& r);
+
+ std::size_t _nxt;
+ std::bitset<MYSZ> _bset; // all zeroes by default
+ BitMonitor<NVAL, StopLevel/NVAL, Iter+1> _children;
+ };
+
+ template <std::size_t N, std::size_t Iter>
+ struct BitMonitor<N,N,Iter> {
+ static constexpr std::size_t NVAL = N;
+ static constexpr std::size_t MYSZ = Pow<NVAL,Iter>::value;
+ static constexpr std::size_t PARENTSZ = Pow<NVAL,Iter-1>::value;
+ static constexpr std::size_t TOTALSZ = MYSZ+PARENTSZ;
+ static constexpr std::size_t LEVEL = Iter;
+ static constexpr bool STOP = true;
+ static constexpr std::size_t npos = std::numeric_limits<std::size_t>::max();
+
+ BitMonitor() : _nxt(0) {}
+
+ inline bool any() const { return _nxt != npos; }
+
+ inline std::size_t get_open() {
+ return _nxt;
+ }
+
+ std::size_t find_open(std::size_t pbin) {
+ pbin *= NVAL;
+ for ( std::size_t idx = pbin; idx < pbin+NVAL; ++idx ) {
+ if ( !_bset[idx] ) {
+ _nxt = idx;
+ return _nxt;
+ }
+ } // for
+ return npos; // something bad just happened
+ }
+
+ // calling program can simply check return value here instead of any()
+ inline bool set(std::size_t bit, std::size_t& cnt) {
+ _bset[bit] = true;
+ Zeros::NC_ZERO = Zeros::C_ZERO;
+
+ // make pbin the group that the parent class is monitoring
+ cnt = bit/NVAL;
+ std::size_t pbin = cnt*NVAL;
+ _nxt = npos;
+ for ( std::size_t idx = bit+1; idx < pbin+NVAL; ++idx ) {
+ if ( !_bset[idx] ) {
+ _nxt = idx;
+ return _nxt != npos;
+ }
+ } // for
+ for ( std::size_t idx = pbin; idx < bit; ++idx ) {
+ if ( !_bset[idx] ) {
+ _nxt = idx;
+ return _nxt != npos;
+ }
+ }
+ return _nxt != npos;
+ }
+
+ inline void unset(std::size_t bit, std::size_t& cnt) {
+ _bset[bit] = false;
+ Zeros::NC_ZERO = Zeros::C_ZERO;
+ cnt = bit/NVAL;
+ if ( _nxt == npos ) { _nxt = bit; }
+ }
+
+ inline std::size_t size() const { return _bset.size(); }
+
+ template <std::size_t A, std::size_t B, std::size_t C>
+ friend
+ typename std::enable_if<BitMonitor<A,B,C>::STOP, std::ostream&>::type
+ operator<<(std::ostream& os, const BitMonitor<A,B,C>& r);
+
+ std::size_t _nxt;
+ std::bitset<MYSZ> _bset; // initialized all-zeroes
+ };
+
+ template <std::size_t N, std::size_t M>
+ struct BitMonitor<N,M,0>; // undefined
+ template <std::size_t N>
+ struct BitMonitor<N,N,0>; // undefined
+
+
+ template <std::size_t A, std::size_t B, std::size_t C>
+ typename std::enable_if<!BitMonitor<A,B,C>::STOP, std::ostream&>::type
+ operator<<(std::ostream& os, const BitMonitor<A,B,C>& r) {
+ os << r._bset << std::endl;
+ os << r._children;
+ return os;
+ }
+
+ template <std::size_t A, std::size_t B, std::size_t C>
+ typename std::enable_if<BitMonitor<A,B,C>::STOP, std::ostream&>::type
+ operator<<(std::ostream& os, const BitMonitor<A,B,C>& r) {
+ os << r._bset;
+ return os;
+ }
+
+
+ //==============
+ // PooledMemory
+ //==============
+ template <typename DataType, std::size_t chunksz = 512, bool CallDestruct = false>
+ struct PooledMemory; // atm, chunksz needs to be >=16 and divisible by 8
+
+ template <typename DataType, std::size_t chunksz, bool CallDestruct>
+ struct PooledMemory {
+ typedef DataType type;
+
+ PooledMemory() : _curr(new MemChunk()), _cache(nullptr)
+ {
+ _blocks.insert(_curr);
+ _blockstarts.insert(_curr->_data);
+ _r.insert(std::make_pair(_curr->_data, _curr));
+ }
+
+ template <typename... Args>
+ inline
+ type* construct(Args... parameters) {
+ if ( !_curr->any() ) {
+ if ( _cache )
+ _curr = _cache;
+ else
+ _curr = new MemChunk();
+
+ _cache = nullptr;
+ _blocks.insert(_curr);
+ _blockstarts.insert(_curr->_data);
+ _r.insert(std::make_pair(_curr->_data, _curr));
+ }
+ return _curr->add(parameters...);
+ }
+
+ inline
+ void release(type* b) {
+ if ( 1 == _blockstarts.size() ) {
+ _curr->remove(b);
+ } else { // possible Chunk removal
+ auto iter = _blockstarts.begin();
+ MemChunk* m;
+ if ( _blockstarts.size() < 10 ) {
+ auto miter = _r.begin();
+ while ( *iter > b ) { ++iter; ++miter; }
+ m = miter->second;
+ } else {
+ iter = _blockstarts.lower_bound(b);
+ m = _r[*iter];
+ }
+ m->remove(b);
+ if ( m->empty() ) {
+ _blocks.erase(m);
+ _blockstarts.erase(iter);
+ _r.erase(*iter);
+ if ( !_cache )
+ _cache = m;
+ else
+ delete m;
+ }
+ }
+ }
+
+ ~PooledMemory() {
+ for ( auto b : _blocks )
+ delete b;
+ if ( _cache )
+ delete _cache;
+ }
+
+ private:
+ template <std::size_t basesz, std::size_t nelements>
+ struct Chunk {
+ Chunk() : _any(true), _cntr(0), _data{} { }
+
+ inline bool any() const { return _any; } // any positions available?
+ inline bool empty() const { return _cntr == 0; } // nothing set
+
+ template <typename... Args>
+ inline type* add(Args... parameters) {
+ std::size_t trackpos = _tracker.get_open();
+ type* address = static_cast<type*>(_data+trackpos);
+ if ( CallDestruct && address ) { address->~type(); }
+ _any &= _tracker.set(trackpos);
+ ++_cntr;
+ return new(address) type(parameters...);
+ }
+
+ inline void remove(type* bt) {
+ // lazy deletion (see add())
+ _tracker.unset(bt-_data);
+ _any = true;
+ --_cntr;
+ }
+
+ ~Chunk() {
+ if ( CallDestruct ) {
+ for ( std::size_t i = 0; i < _tracker.size(); ++i ) {
+ type* address = static_cast<type*>(_data+i);
+ if ( address ) { address->~type(); }
+ } // for
+ }
+ }
+
+ bool _any;
+ std::size_t _cntr;
+ type _data[nelements];
+ //BitMonitor<basesz, nelements> _tracker; // nelements == nbits monitored
+ BitMonitor2<basesz, nelements> _tracker; // nelements == nbits monitored
+ };
+
+ private:
+ static constexpr std::size_t BaseSize = std::numeric_limits<unsigned char>::digits;
+ static constexpr std::size_t NBits = Pow<BaseSize, IntLogN<BaseSize, chunksz>::value>::value;
+ typedef Chunk<BaseSize, NBits> MemChunk;
+
+ MemChunk* _curr;
+ MemChunk* _cache;
+ // Important to remember that _blocks, _blockstarts, _r all have the same #elements
+ std::set<MemChunk*, std::greater<MemChunk*>> _blocks;
+ std::set<type*, std::greater<type*>> _blockstarts;
+ std::map<type*, MemChunk*, std::greater<type*>> _r;
+ };
+
+} // namespace Ext
+
+#endif // __UTILS_SIMPLE_SMALLOBJ_MEM__
diff --git a/interfaces/general-headers/utility/PrintTypes.hpp b/interfaces/general-headers/utility/PrintTypes.hpp
index e469d57..fee58b7 100644
--- a/interfaces/general-headers/utility/PrintTypes.hpp
+++ b/interfaces/general-headers/utility/PrintTypes.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/utility/SingletonType.hpp b/interfaces/general-headers/utility/SingletonType.hpp
index c7ff3c5..a97c5a3 100755
--- a/interfaces/general-headers/utility/SingletonType.hpp
+++ b/interfaces/general-headers/utility/SingletonType.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/general-headers/utility/Typify.hpp b/interfaces/general-headers/utility/Typify.hpp
index 3a3ac24..435b6f8 100755
--- a/interfaces/general-headers/utility/Typify.hpp
+++ b/interfaces/general-headers/utility/Typify.hpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/src/algorithm/sweep/WindowSweepImpl.cpp b/interfaces/src/algorithm/sweep/WindowSweepImpl.cpp
index 27f80ee..2c6e941 100644
--- a/interfaces/src/algorithm/sweep/WindowSweepImpl.cpp
+++ b/interfaces/src/algorithm/sweep/WindowSweepImpl.cpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/src/algorithm/sweep/WindowSweepImpl.specialize.cpp b/interfaces/src/algorithm/sweep/WindowSweepImpl.specialize.cpp
index 3f3d9d4..16b1b25 100644
--- a/interfaces/src/algorithm/sweep/WindowSweepImpl.specialize.cpp
+++ b/interfaces/src/algorithm/sweep/WindowSweepImpl.specialize.cpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/src/data/measurement/NaN.cpp b/interfaces/src/data/measurement/NaN.cpp
index 45dcf2a..9ec82d5 100644
--- a/interfaces/src/data/measurement/NaN.cpp
+++ b/interfaces/src/data/measurement/NaN.cpp
@@ -4,7 +4,7 @@
*/
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/src/data/starch/starchBase64Coding.c b/interfaces/src/data/starch/starchBase64Coding.c
index 19b4aec..d85b9bd 100644
--- a/interfaces/src/data/starch/starchBase64Coding.c
+++ b/interfaces/src/data/starch/starchBase64Coding.c
@@ -6,7 +6,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/src/data/starch/starchConstants.c b/interfaces/src/data/starch/starchConstants.c
index 4c4477e..f837935 100644
--- a/interfaces/src/data/starch/starchConstants.c
+++ b/interfaces/src/data/starch/starchConstants.c
@@ -6,7 +6,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/src/data/starch/starchFileHelpers.c b/interfaces/src/data/starch/starchFileHelpers.c
index 88dc1cb..72dd914 100644
--- a/interfaces/src/data/starch/starchFileHelpers.c
+++ b/interfaces/src/data/starch/starchFileHelpers.c
@@ -6,7 +6,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
diff --git a/interfaces/src/data/starch/starchHelpers.c b/interfaces/src/data/starch/starchHelpers.c
index 478f5c5..8be5078 100644
--- a/interfaces/src/data/starch/starchHelpers.c
+++ b/interfaces/src/data/starch/starchHelpers.c
@@ -6,7 +6,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -246,7 +246,7 @@ STARCH_createTransformTokens(const char *s, const char delim, char **chr, int64_
charCnt = 0U;
sCnt = 0U;
- elemCnt = -1;
+ elemCnt = 0U;
do {
buffer[charCnt++] = s[sCnt];
@@ -254,7 +254,6 @@ STARCH_createTransformTokens(const char *s, const char delim, char **chr, int64_
if (elemCnt < 3) {
buffer[(charCnt - 1)] = '\0';
charCnt = 0;
- elemCnt++;
}
#ifdef DEBUG
fprintf(stderr, "\t--- s [%s] buffer [%s], charCnt [%u], strlen(buffer) [%zu], sCnt [%u], strlen(s) [%zu]\n", s, buffer, charCnt, strlen(buffer), sCnt, strlen(s));
@@ -314,7 +313,7 @@ STARCH_createTransformTokens(const char *s, const char delim, char **chr, int64_
fprintf(stderr, "\t--- copying chromosome field ---\n");
#endif
if (strlen(buffer) > TOKEN_CHR_MAX_LENGTH) {
- fprintf(stderr, "ERROR: Chromosome field length is too long (must be no longer than %ld characters)\n", TOKEN_CHR_MAX_LENGTH);
+ fprintf(stderr, "ERROR: Chromosome field length is too long (must be no longer than %d characters)\n", TOKEN_CHR_MAX_LENGTH);
return STARCH_FATAL_ERROR;
}
/* copy element to chromosome variable, if memory is available */
@@ -339,7 +338,7 @@ STARCH_createTransformTokens(const char *s, const char delim, char **chr, int64_
fprintf(stderr, "\t--- copying whole line ---\n");
#endif
if (strlen(s) > TOKENS_HEADER_MAX_LENGTH) {
- fprintf(stderr, "ERROR: Comment line length is too long (must be no longer than %ld characters)\n", TOKEN_CHR_MAX_LENGTH);
+ fprintf(stderr, "ERROR: Comment line length is too long (must be no longer than %d characters)\n", TOKEN_CHR_MAX_LENGTH);
return STARCH_FATAL_ERROR;
}
/* copy whole line to chromosome variable, if memory is available */
@@ -414,9 +413,10 @@ STARCH_createTransformTokens(const char *s, const char delim, char **chr, int64_
break;
}
/* just keep filling the buffer until we reach s[sCnt]'s null -- we do tests later */
- case 3:
+ case 3:
break;
}
+ elemCnt++;
}
} while (s[sCnt++] != '\0');
@@ -425,14 +425,14 @@ STARCH_createTransformTokens(const char *s, const char delim, char **chr, int64_
/* test id field length */
while ((buffer[idIdx] != delim) && (idIdx++ < TOKEN_ID_MAX_LENGTH)) {}
if (idIdx == TOKEN_ID_MAX_LENGTH) {
- fprintf(stderr, "ERROR: Id field is too long (must be less than %ld characters long)\n", TOKEN_ID_MAX_LENGTH);
+ fprintf(stderr, "ERROR: Id field is too long (must be less than %d characters long)\n", TOKEN_ID_MAX_LENGTH);
return STARCH_FATAL_ERROR;
}
/* test remnant of buffer, if there is more to look at */
if (charCnt > idIdx) {
while ((buffer[idIdx++] != '\0') && (restIdx++ < TOKEN_REST_MAX_LENGTH)) {}
if (restIdx == TOKEN_REST_MAX_LENGTH) {
- fprintf(stderr, "ERROR: Remainder of BED input after id field is too long (must be less than %ld characters long)\n", TOKEN_REST_MAX_LENGTH);
+ fprintf(stderr, "ERROR: Remainder of BED input after id field is too long (must be less than %d characters long)\n", TOKEN_REST_MAX_LENGTH);
return STARCH_FATAL_ERROR;
}
}
@@ -489,7 +489,7 @@ STARCH_createTransformTokensForHeaderlessInput(const char *s, const char delim,
#endif
/* test if element string is longer than allowed bounds */
if (strlen(buffer) > TOKEN_CHR_MAX_LENGTH) {
- fprintf(stderr, "ERROR: Chromosome field length is too long (must be no longer than %ld characters)\n", TOKEN_CHR_MAX_LENGTH);
+ fprintf(stderr, "ERROR: Chromosome field length is too long (must be no longer than %d characters)\n", TOKEN_CHR_MAX_LENGTH);
return STARCH_FATAL_ERROR;
}
/* copy element to chromosome variable, if memory is available */
@@ -596,14 +596,14 @@ STARCH_createTransformTokensForHeaderlessInput(const char *s, const char delim,
/* test id field length */
while ((buffer[idIdx] != delim) && (idIdx++ < TOKEN_ID_MAX_LENGTH)) {}
if (idIdx == TOKEN_ID_MAX_LENGTH) {
- fprintf(stderr, "ERROR: Id field is too long (must be less than %ld characters long)\n", TOKEN_ID_MAX_LENGTH);
+ fprintf(stderr, "ERROR: Id field is too long (must be less than %d characters long)\n", TOKEN_ID_MAX_LENGTH);
return STARCH_FATAL_ERROR;
}
/* test remnant ("rest") of buffer, if there is more to look at */
if (charCnt > idIdx) {
while ((buffer[idIdx++] != '\0') && (restIdx++ < TOKEN_REST_MAX_LENGTH)) {}
if (restIdx == TOKEN_REST_MAX_LENGTH) {
- fprintf(stderr, "ERROR: Remainder of BED input after id field is too long (must be less than %ld characters long)\n", TOKEN_REST_MAX_LENGTH);
+ fprintf(stderr, "ERROR: Remainder of BED input after id field is too long (must be less than %d characters long)\n", TOKEN_REST_MAX_LENGTH);
return STARCH_FATAL_ERROR;
}
}
@@ -706,14 +706,13 @@ STARCH_transformInput(Metadata **md, const FILE *fp, const CompressionType type,
lineIdx++;
buffer[cIdx] = '\0';
if (STARCH_createTransformTokens(buffer, '\t', &chromosome, &start, &stop, &remainder, &lineType) == 0) {
- /*
- Either previous chromosome is NULL, or current chromosome does
- not equal previous chromosome, but the line must be of the
- type 'kBedLineCoordinates' (cf. 'starchMetadataHelpers.h')
+ /*
+ Either previous chromosome is NULL, or current chromosome does
+ not equal previous chromosome, but the line must be of the
+ type 'kBedLineCoordinates' (cf. 'starchMetadataHelpers.h')
*/
-
if ( (lineType == kBedLineCoordinates) && ((!prevChromosome) || (strcmp(chromosome, prevChromosome) != 0)) ) {
- /* close old output file pointer */
+ /* close old output file pointer */
if (outFnPtr != NULL) {
fclose(outFnPtr);
outFnPtr = NULL;
@@ -726,7 +725,7 @@ STARCH_transformInput(Metadata **md, const FILE *fp, const CompressionType type,
return STARCH_FATAL_ERROR;
}
#else
- if (STARCH_compressFileWithBzip2((const char *)outFn, &outCompressedFn, (off_t *) &outCompressedFnSize ) != 0) {
+ if (STARCH_compressFileWithBzip2((const char *)outFn, &outCompressedFn, (off_t *) &outCompressedFnSize ) != 0) {
fprintf(stderr, "ERROR: Could not bzip2 compress per-chromosome output file %s\n", outFn);
return STARCH_FATAL_ERROR;
}
@@ -740,8 +739,8 @@ STARCH_transformInput(Metadata **md, const FILE *fp, const CompressionType type,
return STARCH_FATAL_ERROR;
}
#else
- if (STARCH_compressFileWithGzip((const char*) outFn, &outCompressedFn, (off_t *) &outCompressedFnSize ) != 0) {
- fprintf(stderr, "ERROR: Could not gzip compress per-chromosome output file %s\n", outFn);
+ if (STARCH_compressFileWithGzip((const char*) outFn, &outCompressedFn, (off_t *) &outCompressedFnSize ) != 0) {
+ fprintf(stderr, "ERROR: Could not gzip compress per-chromosome output file %s\n", outFn);
return STARCH_FATAL_ERROR;
}
#endif
@@ -769,7 +768,9 @@ STARCH_transformInput(Metadata **md, const FILE *fp, const CompressionType type,
totalNonUniqueBases,
totalUniqueBases,
duplicateElementExistsFlag,
- nestedElementExistsFlag) != STARCH_EXIT_SUCCESS) {
+ nestedElementExistsFlag,
+ NULL,
+ STARCH_DEFAULT_LINE_STRING_LENGTH) != STARCH_EXIT_SUCCESS) {
fprintf(stderr, "ERROR: Could not update metadata%s\n", outFn);
return STARCH_FATAL_ERROR;
}
@@ -781,12 +782,12 @@ STARCH_transformInput(Metadata **md, const FILE *fp, const CompressionType type,
/* test if current chromosome is already a Metadata record */
#ifdef __cplusplus
if (STARCH_chromosomeInMetadataRecords(reinterpret_cast<const Metadata *>( *md ), chromosome) == STARCH_EXIT_SUCCESS) {
- fprintf(stderr, "ERROR: Found same chromosome in earlier portion of file. Possible interleaving issue? Be sure to first sort input with sort-bed or remove --do-not-sort option from conversion script.\n");
+ fprintf(stderr, "ERROR: Found same chromosome in earlier portion of file. Possible interleaving issue?\nBe sure to first sort input with sort-bed or remove --do-not-sort option from conversion script.\n");
return STARCH_FATAL_ERROR;
}
#else
if (STARCH_chromosomeInMetadataRecords((const Metadata *)*md, chromosome) == STARCH_EXIT_SUCCESS) {
- fprintf(stderr, "ERROR: Found same chromosome in earlier portion of file. Possible interleaving issue? Be sure to first sort input with sort-bed or remove --do-not-sort option from conversion script.\n");
+ fprintf(stderr, "ERROR: Found same chromosome in earlier portion of file. Possible interleaving issue?\nBe sure to first sort input with sort-bed or remove --do-not-sort option from conversion script.\n");
return STARCH_FATAL_ERROR;
}
#endif
@@ -819,7 +820,9 @@ STARCH_transformInput(Metadata **md, const FILE *fp, const CompressionType type,
totalNonUniqueBases,
totalUniqueBases,
duplicateElementExistsFlag,
- nestedElementExistsFlag);
+ nestedElementExistsFlag,
+ NULL,
+ STARCH_DEFAULT_LINE_STRING_LENGTH);
firstRecord = *md;
}
else {
@@ -831,7 +834,9 @@ STARCH_transformInput(Metadata **md, const FILE *fp, const CompressionType type,
totalNonUniqueBases,
totalUniqueBases,
duplicateElementExistsFlag,
- nestedElementExistsFlag);
+ nestedElementExistsFlag,
+ NULL,
+ STARCH_DEFAULT_LINE_STRING_LENGTH);
}
/* make previous chromosome the current chromosome */
@@ -999,7 +1004,9 @@ STARCH_transformInput(Metadata **md, const FILE *fp, const CompressionType type,
totalNonUniqueBases,
totalUniqueBases,
duplicateElementExistsFlag,
- nestedElementExistsFlag);
+ nestedElementExistsFlag,
+ NULL,
+ STARCH_DEFAULT_LINE_STRING_LENGTH);
free(outCompressedFn); outCompressedFn = NULL;
}
@@ -1079,7 +1086,7 @@ STARCH_transformInput(Metadata **md, const FILE *fp, const CompressionType type,
return 0;
}
-int
+int
STARCH_transformHeaderlessInput(Metadata **md, const FILE *fp, const CompressionType type, const char *tag, const Boolean finalizeFlag, const char *note)
{
#ifdef DEBUG
@@ -1189,7 +1196,9 @@ STARCH_transformHeaderlessInput(Metadata **md, const FILE *fp, const Compression
totalNonUniqueBases,
totalUniqueBases,
duplicateElementExistsFlag,
- nestedElementExistsFlag) != STARCH_EXIT_SUCCESS) {
+ nestedElementExistsFlag,
+ NULL,
+ STARCH_DEFAULT_LINE_STRING_LENGTH) != STARCH_EXIT_SUCCESS) {
fprintf(stderr, "ERROR: Could not update metadata%s\n", outFn);
return STARCH_FATAL_ERROR;
}
@@ -1204,7 +1213,7 @@ STARCH_transformHeaderlessInput(Metadata **md, const FILE *fp, const Compression
#else
if (STARCH_chromosomeInMetadataRecords((const Metadata *)*md, chromosome) == STARCH_EXIT_SUCCESS) {
#endif
- fprintf(stderr, "ERROR: Found same chromosome in earlier portion of file. Possible interleaving issue? Be sure to first sort input with sort-bed or remove --do-not-sort option from conversion script.\n");
+ fprintf(stderr, "ERROR: Found same chromosome in earlier portion of file. Possible interleaving issue?\nBe sure to first sort input with sort-bed or remove --do-not-sort option from conversion script.\n");
return STARCH_FATAL_ERROR;
}
@@ -1236,7 +1245,9 @@ STARCH_transformHeaderlessInput(Metadata **md, const FILE *fp, const Compression
totalNonUniqueBases,
totalUniqueBases,
duplicateElementExistsFlag,
- nestedElementExistsFlag);
+ nestedElementExistsFlag,
+ NULL,
+ STARCH_DEFAULT_LINE_STRING_LENGTH);
firstRecord = *md;
}
else {
@@ -1248,7 +1259,9 @@ STARCH_transformHeaderlessInput(Metadata **md, const FILE *fp, const Compression
totalNonUniqueBases,
totalUniqueBases,
duplicateElementExistsFlag,
- nestedElementExistsFlag);
+ nestedElementExistsFlag,
+ NULL,
+ STARCH_DEFAULT_LINE_STRING_LENGTH);
}
/* make previous chromosome the current chromosome */
@@ -1418,7 +1431,9 @@ STARCH_transformHeaderlessInput(Metadata **md, const FILE *fp, const Compression
totalNonUniqueBases,
totalUniqueBases,
duplicateElementExistsFlag,
- nestedElementExistsFlag);
+ nestedElementExistsFlag,
+ NULL,
+ STARCH_DEFAULT_LINE_STRING_LENGTH);
free(outCompressedFn); outCompressedFn = NULL;
free(outFn); outFn = NULL;
@@ -1587,7 +1602,7 @@ STARCH_strndup(const char *s, size_t n)
}
int
-STARCH2_transformInput(unsigned char **header, Metadata **md, const FILE *inFp, const CompressionType compressionType, const char *tag, const char *note, const Boolean headerFlag)
+STARCH2_transformInput(unsigned char **header, Metadata **md, const FILE *inFp, const CompressionType compressionType, const char *tag, const char *note, const Boolean generatePerChrSignatureFlag, const Boolean headerFlag, const Boolean reportProgressFlag, const LineCountType reportProgressN)
{
#ifdef DEBUG
fprintf(stderr, "\n--- STARCH2_transformInput() ---\n");
@@ -1638,13 +1653,13 @@ STARCH2_transformInput(unsigned char **header, Metadata **md, const FILE *inFp,
}
if (headerFlag == kStarchFalse) {
- if (STARCH2_transformHeaderlessBEDInput(inFp, md, compressionType, tag, note) != STARCH_EXIT_SUCCESS) {
+ if (STARCH2_transformHeaderlessBEDInput(inFp, md, compressionType, tag, note, generatePerChrSignatureFlag, reportProgressFlag, reportProgressN) != STARCH_EXIT_SUCCESS) {
fprintf(stderr, "ERROR: Could not write transformed/compressed data to output file pointer.\n");
return STARCH_EXIT_FAILURE;
}
}
else {
- if (STARCH2_transformHeaderedBEDInput(inFp, md, compressionType, tag, note) != STARCH_EXIT_SUCCESS) {
+ if (STARCH2_transformHeaderedBEDInput(inFp, md, compressionType, tag, note, generatePerChrSignatureFlag, reportProgressFlag, reportProgressN) != STARCH_EXIT_SUCCESS) {
fprintf(stderr, "ERROR: Could not write transformed/compressed data to output file pointer.\n");
return STARCH_EXIT_FAILURE;
}
@@ -1672,7 +1687,7 @@ STARCH2_transformInput(unsigned char **header, Metadata **md, const FILE *inFp,
}
int
-STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const CompressionType compressionType, const char *tag, const char *note)
+STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const CompressionType compressionType, const char *tag, const char *note, const Boolean generatePerChrSignatureFlag, const Boolean reportProgressFlag, const LineCountType reportProgressN)
{
#ifdef DEBUG
fprintf(stderr, "\n--- STARCH2_transformHeaderedBEDInput() ---\n");
@@ -1687,6 +1702,7 @@ STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const Compres
int64_t stop = 0;
int64_t pStart = -1;
int64_t pStop = -1;
+ char *pRemainder = NULL;
int64_t previousStop = 0;
int64_t lastPosition = 0;
int64_t lcDiff = 0;
@@ -1708,7 +1724,7 @@ STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const Compres
Metadata *firstRecord = NULL;
char *json = NULL;
CompressionType type = compressionType;
- unsigned char sha1Digest[STARCH2_MD_FOOTER_SHA1_LENGTH];
+ unsigned char sha1Digest[STARCH2_MD_FOOTER_SHA1_LENGTH] = {0};
char *base64EncodedSha1Digest = NULL;
int zError = -1;
char zBuffer[STARCH_Z_BUFFER_MAX_LENGTH] = {0};
@@ -1727,6 +1743,9 @@ STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const Compres
Boolean nonCoordLineBufNeedsPrinting = kStarchFalse;
char const *nullChr = "null";
char const *nullCompressedFn = "null";
+ char const *nullSig = "null";
+ struct sha1_ctx perChromosomeHashCtx;
+ LineLengthType maxStringLength = STARCH_DEFAULT_LINE_STRING_LENGTH;
/* increment total file size by header bytes */
#ifdef DEBUG
@@ -1808,6 +1827,11 @@ STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const Compres
}
}
+ if (generatePerChrSignatureFlag) {
+ /* set up per-chromosome hash context */
+ sha1_init_ctx(&perChromosomeHashCtx);
+ }
+
/* fill up a "transformation" buffer with data and then compress it */
#ifdef __cplusplus
while ((c = fgetc(const_cast<FILE *>( inFp ))) != EOF) {
@@ -1819,25 +1843,70 @@ STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const Compres
if (c == '\n') {
lineIdx++;
untransformedBuffer[cIdx] = '\0';
+ maxStringLength = (maxStringLength >= cIdx) ? maxStringLength : cIdx;
+
+ if (STARCH_createTransformTokens(untransformedBuffer, '\t', &chromosome, &start, &stop, &remainder, &lineType) == 0) {
+ if (pRemainder) {
+ /* if previous start and stop coordinates are the same, compare the remainder here */
+ if ((start == pStart) && (stop == pStop) && (strcmp(remainder, pRemainder) < 0)) {
+ fprintf(stderr, "ERROR: Elements with same start and stop coordinates have remainders in wrong sort order.\nBe sure to first sort input with sort-bed or remove --do-not-sort option from conversion script.\n");
+ return STARCH_FATAL_ERROR;
+ }
+ free(pRemainder), pRemainder = NULL;
+ }
+
+ if ((reportProgressFlag == kStarchTrue) && (lineIdx % reportProgressN == 0)) {
+ fprintf(stderr, "PROGRESS: Transforming element [%ld] of chromosome [%s] -> [%s]\n", lineIdx, chromosome, untransformedBuffer);
+ }
- if (STARCH_createTransformTokens(untransformedBuffer, '\t', &chromosome, &start, &stop, &remainder, &lineType) == 0)
- {
if ( (lineType == kBedLineCoordinates) && ((!prevChromosome) || (strcmp(chromosome, prevChromosome) != 0)) )
{
if (prevChromosome)
{
#ifdef __cplusplus
- if (STARCH_chromosomeInMetadataRecords(reinterpret_cast<const Metadata *>( firstRecord ), chromosome) == STARCH_EXIT_SUCCESS) {
+ if (STARCH_chromosomeInMetadataRecords(reinterpret_cast<const Metadata *>( firstRecord ), chromosome) == STARCH_EXIT_SUCCESS)
+#else
+ if (STARCH_chromosomeInMetadataRecords((const Metadata *)firstRecord, chromosome) == STARCH_EXIT_SUCCESS)
+#endif
+ {
+ fprintf(stderr, "ERROR: Found same chromosome in earlier portion of file. Possible interleaving issue?\nBe sure to first sort input with sort-bed or remove --do-not-sort option from conversion script.\n");
+ return STARCH_FATAL_ERROR;
+ }
+#ifdef __cplusplus
+ if (STARCH_chromosomePositionedBeforeExistingMetadataRecord(reinterpret_cast<const Metadata *>( firstRecord ), chromosome) == STARCH_EXIT_SUCCESS)
#else
- if (STARCH_chromosomeInMetadataRecords((const Metadata *)firstRecord, chromosome) == STARCH_EXIT_SUCCESS) {
+ if (STARCH_chromosomePositionedBeforeExistingMetadataRecord((const Metadata *)firstRecord, chromosome) == STARCH_EXIT_SUCCESS)
#endif
- fprintf(stderr, "ERROR: Found same chromosome in earlier portion of file. Possible interleaving issue? Be sure to first sort input with sort-bed or remove --do-not-sort option from conversion script.\n");
+ {
+ fprintf(stderr, "ERROR: Chromosome name not ordered lexicographically. Possible sorting issue?\nBe sure to first sort input with sort-bed or remove --do-not-sort option from conversion script.\n");
return STARCH_FATAL_ERROR;
}
sprintf(compressedFn, "%s.%s", prevChromosome, tag);
#ifdef DEBUG
fprintf(stderr, "\t(final-between-chromosome) transformedBuffer:\n%s\n\t\tintermediateBuffer:\n%s\n", transformedBuffer, intermediateBuffer);
#endif
+ if (generatePerChrSignatureFlag) {
+ /* hash the transformed buffer */
+ sha1_process_bytes(transformedBuffer, currentTransformedBufferLength, &perChromosomeHashCtx);
+ sha1_finish_ctx(&perChromosomeHashCtx, sha1Digest);
+#ifdef __cplusplus
+ STARCH_encodeBase64(&base64EncodedSha1Digest,
+ static_cast<size_t>( STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH ),
+ reinterpret_cast<const unsigned char *>( sha1Digest ),
+ static_cast<size_t>( STARCH2_MD_FOOTER_SHA1_LENGTH ) );
+#else
+ STARCH_encodeBase64(&base64EncodedSha1Digest,
+ (const size_t) STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH,
+ (const unsigned char *) sha1Digest,
+ (const size_t) STARCH2_MD_FOOTER_SHA1_LENGTH);
+#endif
+#ifdef DEBUG
+ fprintf(stderr, "\nPROGRESS: SHA-1 digest for chr [%s] is [%s]\n", prevChromosome, base64EncodedSha1Digest);
+#endif
+
+ sha1_init_ctx(&perChromosomeHashCtx);
+ }
+
if (compressionType == kBzip2) {
#ifdef DEBUG
fprintf(stderr, "\t(final-between-chromosome) finalizing current chromosome: %s\n", prevChromosome);
@@ -1900,7 +1969,9 @@ STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const Compres
totalNonUniqueBases,
totalUniqueBases,
duplicateElementExistsFlag,
- nestedElementExistsFlag) != STARCH_EXIT_SUCCESS) {
+ nestedElementExistsFlag,
+ base64EncodedSha1Digest,
+ maxStringLength) != STARCH_EXIT_SUCCESS) {
fprintf(stderr, "ERROR: Could not update metadata %s\n", compressedFn);
return STARCH_FATAL_ERROR;
}
@@ -2010,7 +2081,9 @@ STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const Compres
totalNonUniqueBases,
totalUniqueBases,
duplicateElementExistsFlag,
- nestedElementExistsFlag) != STARCH_EXIT_SUCCESS) {
+ nestedElementExistsFlag,
+ base64EncodedSha1Digest,
+ maxStringLength) != STARCH_EXIT_SUCCESS) {
fprintf(stderr, "ERROR: Could not update metadata %s\n", compressedFn);
return STARCH_FATAL_ERROR;
}
@@ -2048,6 +2121,10 @@ STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const Compres
fprintf(stderr, "\t(final-between-chromosome) initialized z-stream\n");
#endif
}
+
+ /* clean up per-chromosome hash digest */
+ if (base64EncodedSha1Digest)
+ free(base64EncodedSha1Digest), base64EncodedSha1Digest = NULL;
}
/* create placeholder records at current chromosome */
@@ -2064,7 +2141,9 @@ STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const Compres
0UL,
0UL,
STARCH_DEFAULT_DUPLICATE_ELEMENT_FLAG_VALUE,
- STARCH_DEFAULT_NESTED_ELEMENT_FLAG_VALUE);
+ STARCH_DEFAULT_NESTED_ELEMENT_FLAG_VALUE,
+ base64EncodedSha1Digest,
+ maxStringLength);
if (!*md) {
fprintf(stderr, "ERROR: Not enough memory is available\n");
return STARCH_EXIT_FAILURE;
@@ -2080,7 +2159,9 @@ STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const Compres
0UL,
0UL,
STARCH_DEFAULT_DUPLICATE_ELEMENT_FLAG_VALUE,
- STARCH_DEFAULT_NESTED_ELEMENT_FLAG_VALUE);
+ STARCH_DEFAULT_NESTED_ELEMENT_FLAG_VALUE,
+ base64EncodedSha1Digest,
+ maxStringLength);
}
/* make previous chromosome the current chromosome */
@@ -2111,6 +2192,7 @@ STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const Compres
lastPosition = 0;
pStart = -1;
pStop = -1;
+ if (pRemainder) { free(pRemainder), pRemainder = NULL; }
previousStop = 0;
lcDiff = 0;
lineIdx = 0UL;
@@ -2122,6 +2204,7 @@ STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const Compres
currentRecSize = 0UL;
transformedBuffer[currentTransformedBufferLength] = '\0';
currentTransformedBufferLength = 0U;
+ maxStringLength = STARCH_DEFAULT_LINE_STRING_LENGTH;
}
else if (lineType == kBedLineCoordinates)
withinChr = kStarchTrue;
@@ -2339,6 +2422,12 @@ STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const Compres
/* set pElement values */
pStart = start;
pStop = stop;
+ if (pRemainder) {
+ free(pRemainder);
+ pRemainder = NULL;
+ }
+ if (remainder)
+ pRemainder = STARCH_strndup(remainder, strlen(remainder));
}
if (withinChr == kStarchTrue)
@@ -2360,6 +2449,11 @@ STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const Compres
if (cIdx > 0) {
untransformedBuffer[cIdx] = '\0';
+#ifdef __cplusplus
+ maxStringLength = (maxStringLength >= static_cast<LineLengthType>(strlen(untransformedBuffer))) ? maxStringLength : static_cast<LineLengthType>(strlen(untransformedBuffer));
+#else
+ maxStringLength = (maxStringLength >= (LineLengthType) strlen(untransformedBuffer)) ? maxStringLength : (LineLengthType) strlen(untransformedBuffer);
+#endif
if (STARCH_createTransformTokensForHeaderlessInput(untransformedBuffer, '\t', &chromosome, &start, &stop, &remainder) == 0) {
#ifdef DEBUG
fprintf(stderr, "\t(just-before-last-pass) untransformedBuffer:\n%s\n", untransformedBuffer);
@@ -2438,6 +2532,15 @@ STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const Compres
/* test for nested element */
if ((pStart < start) && (pStop > stop))
nestedElementExistsFlag = kStarchTrue;
+
+ if (pRemainder) {
+ /* if previous start and stop coordinates are the same, compare the remainder here */
+ if ((start == pStart) && (stop == pStop) && (strcmp(remainder, pRemainder) < 0)) {
+ fprintf(stderr, "ERROR: Elements with same start and stop coordinates have remainders in wrong sort order.\nBe sure to first sort input with sort-bed or remove --do-not-sort option from conversion script.\n");
+ return STARCH_FATAL_ERROR;
+ }
+ free(pRemainder), pRemainder = NULL;
+ }
}
else {
fprintf(stderr, "ERROR: BED data is corrupt at line %lu\n", lineIdx);
@@ -2450,8 +2553,28 @@ STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const Compres
#ifdef DEBUG
fprintf(stderr, "\t(last-pass) transformedBuffer:\n%s\n\t\tintermediateBuffer:\n%s\n", transformedBuffer, intermediateBuffer);
- /*fprintf(stderr, "\t(last-pass) to be compressed - transformedBuffer:\n%s\n", transformedBuffer);*/
#endif
+
+ if (generatePerChrSignatureFlag) {
+ /* hash the transformed buffer */
+ sha1_process_bytes(transformedBuffer, currentTransformedBufferLength, &perChromosomeHashCtx);
+ sha1_finish_ctx(&perChromosomeHashCtx, sha1Digest);
+#ifdef __cplusplus
+ STARCH_encodeBase64(&base64EncodedSha1Digest,
+ static_cast<size_t>( STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH ),
+ reinterpret_cast<const unsigned char *>( sha1Digest ),
+ static_cast<size_t>( STARCH2_MD_FOOTER_SHA1_LENGTH ) );
+#else
+ STARCH_encodeBase64(&base64EncodedSha1Digest,
+ (const size_t) STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH,
+ (const unsigned char *) sha1Digest,
+ (const size_t) STARCH2_MD_FOOTER_SHA1_LENGTH);
+#endif
+#ifdef DEBUG
+ fprintf(stderr, "\nPROGRESS: SHA-1 digest for chr [%s] is [%s]\n", prevChromosome, base64EncodedSha1Digest);
+#endif
+ }
+
/* last-pass, bzip2 */
if (compressionType == kBzip2) {
#ifdef DEBUG
@@ -2581,20 +2704,35 @@ STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const Compres
totalNonUniqueBases,
totalUniqueBases,
duplicateElementExistsFlag,
- nestedElementExistsFlag) != STARCH_EXIT_SUCCESS) {
+ nestedElementExistsFlag,
+ base64EncodedSha1Digest,
+ maxStringLength) != STARCH_EXIT_SUCCESS) {
/*
If the stream or input file contains no BED records, then the Metadata pointer md will
be NULL, as will the char pointer prevChromosome. So we put in a stub metadata record.
*/
lineIdx = 0;
*md = NULL;
- *md = STARCH_createMetadata(nullChr, nullCompressedFn, currentRecSize, lineIdx, 0UL, 0UL, STARCH_DEFAULT_DUPLICATE_ELEMENT_FLAG_VALUE, STARCH_DEFAULT_NESTED_ELEMENT_FLAG_VALUE);
+ *md = STARCH_createMetadata(nullChr,
+ nullCompressedFn,
+ currentRecSize,
+ lineIdx,
+ 0UL,
+ 0UL,
+ STARCH_DEFAULT_DUPLICATE_ELEMENT_FLAG_VALUE,
+ STARCH_DEFAULT_NESTED_ELEMENT_FLAG_VALUE,
+ nullSig,
+ 0UL);
if (!*md) {
fprintf(stderr, "ERROR: Not enough memory is available\n");
return STARCH_EXIT_FAILURE;
}
firstRecord = *md;
- }
+ }
+
+ /* clean up per-chromosome hash digest */
+ if (base64EncodedSha1Digest)
+ free(base64EncodedSha1Digest), base64EncodedSha1Digest = NULL;
/* reset metadata pointer */
*md = firstRecord;
@@ -2623,9 +2761,15 @@ STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const Compres
fprintf(stderr, "\tencoding md signature...\n");
#endif
#ifdef __cplusplus
- STARCH_encodeBase64(&base64EncodedSha1Digest, static_cast<size_t>( STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH ), reinterpret_cast<const unsigned char *>( sha1Digest ), static_cast<size_t>( STARCH2_MD_FOOTER_SHA1_LENGTH ));
+ STARCH_encodeBase64(&base64EncodedSha1Digest,
+ static_cast<size_t>( STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH ),
+ reinterpret_cast<const unsigned char *>( sha1Digest ),
+ static_cast<size_t>( STARCH2_MD_FOOTER_SHA1_LENGTH ));
#else
- STARCH_encodeBase64(&base64EncodedSha1Digest, (const size_t) STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH, (const unsigned char *) sha1Digest, (const size_t) STARCH2_MD_FOOTER_SHA1_LENGTH);
+ STARCH_encodeBase64(&base64EncodedSha1Digest,
+ (const size_t) STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH,
+ (const unsigned char *) sha1Digest,
+ (const size_t) STARCH2_MD_FOOTER_SHA1_LENGTH);
#endif
/* build footer */
@@ -2674,7 +2818,7 @@ STARCH2_transformHeaderedBEDInput(const FILE *inFp, Metadata **md, const Compres
}
int
-STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const CompressionType compressionType, const char *tag, const char *note)
+STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const CompressionType compressionType, const char *tag, const char *note, const Boolean generatePerChrSignatureFlag, const Boolean reportProgressFlag, const LineCountType reportProgressN)
{
#ifdef DEBUG
fprintf(stderr, "\n--- STARCH2_transformHeaderlessBEDInput() ---\n");
@@ -2689,6 +2833,7 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
int64_t stop = 0;
int64_t pStart = -1;
int64_t pStop = -1;
+ char *pRemainder = NULL;
int64_t previousStop = 0;
int64_t lastPosition = 0;
int64_t lcDiff = 0;
@@ -2711,7 +2856,7 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
char *json = NULL;
char *jsonCopy = NULL;
CompressionType type = compressionType;
- unsigned char sha1Digest[STARCH2_MD_FOOTER_SHA1_LENGTH];
+ unsigned char sha1Digest[STARCH2_MD_FOOTER_SHA1_LENGTH] = {0};
char *base64EncodedSha1Digest = NULL;
int zError = -1;
char zBuffer[STARCH_Z_BUFFER_MAX_LENGTH] = {0};
@@ -2727,6 +2872,9 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
char footerBuffer[STARCH2_MD_FOOTER_LENGTH] = {0};
char const *nullChr = "null";
char const *nullCompressedFn = "null";
+ char const *nullSig = "null";
+ struct sha1_ctx perChromosomeHashCtx;
+ LineLengthType maxStringLength = STARCH_DEFAULT_LINE_STRING_LENGTH;
/* increment total file size by header bytes */
#ifdef DEBUG
@@ -2808,6 +2956,11 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
}
}
+ if (generatePerChrSignatureFlag) {
+ /* set up per-chromosome hash context */
+ sha1_init_ctx(&perChromosomeHashCtx);
+ }
+
/* fill up a "transformation" buffer with data and then compress it */
#ifdef __cplusplus
while ((c = fgetc(const_cast<FILE *>( inFp ))) != EOF) {
@@ -2819,23 +2972,69 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
if (c == '\n') {
lineIdx++;
untransformedBuffer[cIdx] = '\0';
+ maxStringLength = (maxStringLength >= cIdx) ? maxStringLength : cIdx;
- if (STARCH_createTransformTokensForHeaderlessInput(untransformedBuffer, '\t', &chromosome, &start, &stop, &remainder) == 0)
- {
- if ( (!prevChromosome) || (strcmp(chromosome, prevChromosome) != 0) ) {
- if (prevChromosome) {
+ if (STARCH_createTransformTokensForHeaderlessInput(untransformedBuffer, '\t', &chromosome, &start, &stop, &remainder) == 0) {
+ if (pRemainder) {
+ /* if previous start and stop coordinates are the same, compare the remainder here */
+ if ((start == pStart) && (stop == pStop) && (strcmp(remainder, pRemainder) < 0)) {
+ fprintf(stderr, "ERROR: Elements with same start and stop coordinates have remainders in wrong sort order.\nBe sure to first sort input with sort-bed or remove --do-not-sort option from conversion script.\n");
+ return STARCH_FATAL_ERROR;
+ }
+ free(pRemainder), pRemainder = NULL;
+ }
+
+ if ((reportProgressFlag == kStarchTrue) && (lineIdx % reportProgressN == 0)) {
+ fprintf(stderr, "PROGRESS: Transforming element [%ld] of chromosome [%s] -> [%s]\n", lineIdx, chromosome, untransformedBuffer);
+ }
+
+ if ( (!prevChromosome) || (strcmp(chromosome, prevChromosome) != 0) )
+ {
+ if (prevChromosome)
+ {
#ifdef __cplusplus
- if (STARCH_chromosomeInMetadataRecords(reinterpret_cast<const Metadata *>( firstRecord ), chromosome) == STARCH_EXIT_SUCCESS) {
+ if (STARCH_chromosomeInMetadataRecords(reinterpret_cast<const Metadata *>( firstRecord ), chromosome) == STARCH_EXIT_SUCCESS)
#else
- if (STARCH_chromosomeInMetadataRecords((const Metadata *)firstRecord, chromosome) == STARCH_EXIT_SUCCESS) {
+ if (STARCH_chromosomeInMetadataRecords((const Metadata *)firstRecord, chromosome) == STARCH_EXIT_SUCCESS)
#endif
- fprintf(stderr, "ERROR: Found same chromosome in earlier portion of file. Possible interleaving issue? Be sure to first sort input with sort-bed or remove --do-not-sort option from conversion script.\n");
+ {
+ fprintf(stderr, "ERROR: Found same chromosome in earlier portion of file. Possible interleaving issue?\nBe sure to first sort input with sort-bed or remove --do-not-sort option from conversion script.\n");
+ return STARCH_FATAL_ERROR;
+ }
+#ifdef __cplusplus
+ if (STARCH_chromosomePositionedBeforeExistingMetadataRecord(reinterpret_cast<const Metadata *>( firstRecord ), chromosome) == STARCH_EXIT_SUCCESS)
+#else
+ if (STARCH_chromosomePositionedBeforeExistingMetadataRecord((const Metadata *)firstRecord, chromosome) == STARCH_EXIT_SUCCESS)
+#endif
+ {
+ fprintf(stderr, "ERROR: Chromosome name not ordered lexicographically. Possible sorting issue?\nBe sure to first sort input with sort-bed or remove --do-not-sort option from conversion script.\n");
return STARCH_FATAL_ERROR;
}
sprintf(compressedFn, "%s.%s", prevChromosome, tag);
#ifdef DEBUG
- fprintf(stderr, "\t(final-between-chromosome) transformedBuffer:\n%s\n", transformedBuffer);
+ fprintf(stderr, "\t(final-between-chromosome) transformedBuffer before hash:\n[%s]\n", transformedBuffer);
+#endif
+ if (generatePerChrSignatureFlag) {
+ /* hash the transformed buffer */
+ sha1_process_bytes(transformedBuffer, currentTransformedBufferLength, &perChromosomeHashCtx);
+ sha1_finish_ctx(&perChromosomeHashCtx, sha1Digest);
+#ifdef __cplusplus
+ STARCH_encodeBase64(&base64EncodedSha1Digest,
+ static_cast<size_t>( STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH ),
+ reinterpret_cast<const unsigned char *>( sha1Digest ),
+ static_cast<size_t>( STARCH2_MD_FOOTER_SHA1_LENGTH ) );
+#else
+ STARCH_encodeBase64(&base64EncodedSha1Digest,
+ (const size_t) STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH,
+ (const unsigned char *) sha1Digest,
+ (const size_t) STARCH2_MD_FOOTER_SHA1_LENGTH);
+#endif
+#ifdef DEBUG
+ fprintf(stderr, "\nPROGRESS: SHA-1 digest for chr [%s] is [%s]\n", prevChromosome, base64EncodedSha1Digest);
#endif
+ sha1_init_ctx(&perChromosomeHashCtx);
+ }
+
if (compressionType == kBzip2) {
#ifdef DEBUG
fprintf(stderr, "\t(final-between-chromosome) finalizing current chromosome: %s\n", prevChromosome);
@@ -2898,7 +3097,9 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
totalNonUniqueBases,
totalUniqueBases,
duplicateElementExistsFlag,
- nestedElementExistsFlag) != STARCH_EXIT_SUCCESS) {
+ nestedElementExistsFlag,
+ base64EncodedSha1Digest,
+ maxStringLength) != STARCH_EXIT_SUCCESS) {
fprintf(stderr, "ERROR: Could not update metadata %s\n", compressedFn);
return STARCH_FATAL_ERROR;
}
@@ -3008,7 +3209,9 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
totalNonUniqueBases,
totalUniqueBases,
duplicateElementExistsFlag,
- nestedElementExistsFlag) != STARCH_EXIT_SUCCESS) {
+ nestedElementExistsFlag,
+ base64EncodedSha1Digest,
+ maxStringLength) != STARCH_EXIT_SUCCESS) {
fprintf(stderr, "ERROR: Could not update metadata %s\n", compressedFn);
return STARCH_FATAL_ERROR;
}
@@ -3046,12 +3249,16 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
fprintf(stderr, "\t(final-between-chromosome) initialized z-stream\n");
#endif
}
+
+ /* clean up per-chromosome hash digest */
+ if (base64EncodedSha1Digest)
+ free(base64EncodedSha1Digest), base64EncodedSha1Digest = NULL;
}
/* create placeholder records at current chromosome */
sprintf(compressedFn, "%s.%s", chromosome, tag);
#ifdef DEBUG
- fprintf(stderr, "\t(final-between-chromosome) creating placeholder md record at chromosome: %s (compressedFn: %s)\n", chromosome, compressedFn);
+ fprintf(stderr, "\t(final-between-chromosome) creating placeholder md record at chromosome: %s (compressedFn: %s) (signature: %s)\n", chromosome, compressedFn, base64EncodedSha1Digest);
#endif
if (recIdx == 0) {
*md = NULL;
@@ -3062,7 +3269,9 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
0UL,
0UL,
STARCH_DEFAULT_DUPLICATE_ELEMENT_FLAG_VALUE,
- STARCH_DEFAULT_NESTED_ELEMENT_FLAG_VALUE);
+ STARCH_DEFAULT_NESTED_ELEMENT_FLAG_VALUE,
+ NULL,
+ STARCH_DEFAULT_LINE_STRING_LENGTH);
if (!*md) {
fprintf(stderr, "ERROR: Not enough memory is available\n");
return STARCH_EXIT_FAILURE;
@@ -3078,7 +3287,9 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
0UL,
0UL,
STARCH_DEFAULT_DUPLICATE_ELEMENT_FLAG_VALUE,
- STARCH_DEFAULT_DUPLICATE_ELEMENT_FLAG_VALUE);
+ STARCH_DEFAULT_DUPLICATE_ELEMENT_FLAG_VALUE,
+ NULL,
+ STARCH_DEFAULT_LINE_STRING_LENGTH);
}
/* make previous chromosome the current chromosome */
@@ -3109,6 +3320,7 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
lastPosition = 0;
pStart = -1;
pStop = -1;
+ if (pRemainder) { free(pRemainder), pRemainder = NULL; }
previousStop = 0;
lcDiff = 0;
lineIdx = 0UL;
@@ -3120,6 +3332,7 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
currentRecSize = 0UL;
transformedBuffer[currentTransformedBufferLength] = '\0';
currentTransformedBufferLength = 0U;
+ maxStringLength = STARCH_DEFAULT_LINE_STRING_LENGTH;
}
else
withinChr = kStarchTrue;
@@ -3186,7 +3399,7 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
/* compress transformedBuffer[] and send to stdout */
#ifdef DEBUG
fprintf(stderr, "\t(intermediate) to be compressed -- transformedBuffer:\n%s\n", transformedBuffer);
-#endif
+#endif
if (compressionType == kBzip2) {
#ifdef DEBUG
fprintf(stderr, "\t(intermediate) current chromosome: %s\n", prevChromosome);
@@ -3254,8 +3467,16 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
fwrite(zBuffer, 1, zHave, stdout);
fflush(stdout);
} while (zStream.avail_out == 0);
- }
+ }
+#ifdef DEBUG
+ fprintf(stderr, "\t(intermediate) transformedBuffer before hash:\n[%s]\n", transformedBuffer);
+#endif
+ if (generatePerChrSignatureFlag) {
+ /* hash the transformed buffer */
+ sha1_process_bytes(transformedBuffer, currentTransformedBufferLength, &perChromosomeHashCtx);
+ }
+ /* copy transformed buffer to intermediate buffer */
memcpy(transformedBuffer, intermediateBuffer, strlen(intermediateBuffer) + 1);
currentTransformedBufferLength = strlen(intermediateBuffer);
memset(intermediateBuffer, 0, strlen(intermediateBuffer) + 1);
@@ -3287,14 +3508,14 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
#else
totalUniqueBases += (BaseCountType) (stop - start);
#endif
- }
+ }
else if (previousStop < stop) {
#ifdef __cplusplus
totalUniqueBases += static_cast<BaseCountType>( stop - previousStop );
#else
totalUniqueBases += (BaseCountType) (stop - previousStop);
#endif
- }
+ }
previousStop = (stop > previousStop) ? stop : previousStop;
#ifdef DEBUG
@@ -3305,7 +3526,7 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
fprintf(stderr, "\t(intermediate) duplicateElementExistsFlag: %d\tnestedElementExistsFlag: %d\n", (int) duplicateElementExistsFlag, (int) nestedElementExistsFlag);
#endif
#endif
-
+
/* test for duplicate element */
if ((pStart == start) && (pStop == stop))
duplicateElementExistsFlag = kStarchTrue;
@@ -3317,6 +3538,12 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
/* set pElement values */
pStart = start;
pStop = stop;
+ if (pRemainder) {
+ free(pRemainder);
+ pRemainder = NULL;
+ }
+ if (remainder)
+ pRemainder = STARCH_strndup(remainder, strlen(remainder));
if (withinChr == kStarchTrue)
free(chromosome), chromosome = NULL;
@@ -3337,6 +3564,7 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
if (cIdx > 0) {
untransformedBuffer[cIdx] = '\0';
+ maxStringLength = (maxStringLength >= cIdx) ? maxStringLength : cIdx;
if (STARCH_createTransformTokensForHeaderlessInput(untransformedBuffer, '\t', &chromosome, &start, &stop, &remainder) == 0) {
#ifdef DEBUG
fprintf(stderr, "\t(just-before-last-pass) untransformedBuffer:\n%s\n", untransformedBuffer);
@@ -3397,7 +3625,7 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
#ifdef DEBUG
fprintf(stderr, "\t(intermediate) state of intermediateBuffer before test:\n%s\n", intermediateBuffer);
#endif
-
+
/* append intermediateBuffer to transformedBuffer */
#ifdef DEBUG
fprintf(stderr, "\t(intermediate) appending intermediateBuffer to transformedBuffer\n");
@@ -3419,14 +3647,14 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
#else
totalUniqueBases += (BaseCountType) (stop - start);
#endif
- }
+ }
else if (previousStop < stop) {
#ifdef __cplusplus
totalUniqueBases += static_cast<BaseCountType>( stop - previousStop );
#else
totalUniqueBases += (BaseCountType) (stop - previousStop);
#endif
- }
+ }
previousStop = (stop > previousStop) ? stop : previousStop;
#ifdef DEBUG
@@ -3437,7 +3665,7 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
fprintf(stderr, "\t(intermediate) duplicateElementExistsFlag: %d\tnestedElementExistsFlag: %d\n", (int) duplicateElementExistsFlag, (int) nestedElementExistsFlag);
#endif
#endif
-
+
/* test for duplicate element */
if ((pStart == start) && (pStop == stop))
duplicateElementExistsFlag = kStarchTrue;
@@ -3445,6 +3673,15 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
/* test for nested element */
if ((pStart < start) && (pStop > stop))
nestedElementExistsFlag = kStarchTrue;
+
+ if (pRemainder) {
+ /* if previous start and stop coordinates are the same, compare the remainder here */
+ if ((start == pStart) && (stop == pStop) && (strcmp(remainder, pRemainder) < 0)) {
+ fprintf(stderr, "ERROR: Elements with same start and stop coordinates have remainders in wrong sort order.\nBe sure to first sort input with sort-bed or remove --do-not-sort option from conversion script.\n");
+ return STARCH_FATAL_ERROR;
+ }
+ free(pRemainder), pRemainder = NULL;
+ }
}
else {
fprintf(stderr, "ERROR: BED data is corrupt at line %lu\n", lineIdx);
@@ -3458,6 +3695,31 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
#ifdef DEBUG
fprintf(stderr, "\t(last-pass) transformedBuffer:\n%s\n\t\tintermediateBuffer:\n%s\n", transformedBuffer, intermediateBuffer);
#endif
+
+#ifdef DEBUG
+ fprintf(stderr, "\t(last-pass) transformedBuffer before hash:\n[%s]\n", transformedBuffer);
+#endif
+
+ if (generatePerChrSignatureFlag) {
+ /* hash the transformed buffer */
+ sha1_process_bytes(transformedBuffer, currentTransformedBufferLength, &perChromosomeHashCtx);
+ sha1_finish_ctx(&perChromosomeHashCtx, sha1Digest);
+#ifdef __cplusplus
+ STARCH_encodeBase64(&base64EncodedSha1Digest,
+ static_cast<size_t>( STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH ),
+ reinterpret_cast<const unsigned char *>( sha1Digest ),
+ static_cast<size_t>( STARCH2_MD_FOOTER_SHA1_LENGTH ) );
+#else
+ STARCH_encodeBase64(&base64EncodedSha1Digest,
+ (const size_t) STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH,
+ (const unsigned char *) sha1Digest,
+ (const size_t) STARCH2_MD_FOOTER_SHA1_LENGTH);
+#endif
+#ifdef DEBUG
+ fprintf(stderr, "\nPROGRESS: SHA-1 digest for chr [%s] is [%s]\n", prevChromosome, base64EncodedSha1Digest);
+#endif
+ }
+
/* last-pass, bzip2 */
if (compressionType == kBzip2) {
#ifdef DEBUG
@@ -3593,7 +3855,9 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
totalNonUniqueBases,
totalUniqueBases,
duplicateElementExistsFlag,
- nestedElementExistsFlag) != STARCH_EXIT_SUCCESS) {
+ nestedElementExistsFlag,
+ base64EncodedSha1Digest,
+ maxStringLength) != STARCH_EXIT_SUCCESS) {
/*
If the stream or input file contains no BED records, then the Metadata pointer md will
be NULL, as will the char pointer prevChromosome. So we put in a stub metadata record.
@@ -3607,13 +3871,19 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
0UL,
0UL,
STARCH_DEFAULT_DUPLICATE_ELEMENT_FLAG_VALUE,
- STARCH_DEFAULT_NESTED_ELEMENT_FLAG_VALUE);
+ STARCH_DEFAULT_NESTED_ELEMENT_FLAG_VALUE,
+ nullSig,
+ 0UL);
if (!*md) {
fprintf(stderr, "ERROR: Not enough memory is available\n");
return STARCH_EXIT_FAILURE;
}
firstRecord = *md;
- }
+ }
+
+ /* clean up per-chromosome hash digest */
+ if (base64EncodedSha1Digest)
+ free(base64EncodedSha1Digest), base64EncodedSha1Digest = NULL;
/* reset metadata pointer */
*md = firstRecord;
@@ -3638,18 +3908,24 @@ STARCH2_transformHeaderlessBEDInput(const FILE *inFp, Metadata **md, const Compr
#endif
free(jsonCopy);
- /* encode signature in base64 encoding */
+/* encode signature in base64 encoding */
#ifdef DEBUG
fprintf(stderr, "\tencoding md signature...\n");
#endif
#ifdef __cplusplus
- STARCH_encodeBase64(&base64EncodedSha1Digest, static_cast<size_t>( STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH ), reinterpret_cast<const unsigned char *>( sha1Digest ), static_cast<size_t>( STARCH2_MD_FOOTER_SHA1_LENGTH ) );
+ STARCH_encodeBase64(&base64EncodedSha1Digest,
+ static_cast<size_t>( STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH ),
+ reinterpret_cast<const unsigned char *>( sha1Digest ),
+ static_cast<size_t>( STARCH2_MD_FOOTER_SHA1_LENGTH ) );
#else
- STARCH_encodeBase64(&base64EncodedSha1Digest, (const size_t) STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH, (const unsigned char *) sha1Digest, (const size_t) STARCH2_MD_FOOTER_SHA1_LENGTH);
+ STARCH_encodeBase64(&base64EncodedSha1Digest,
+ (const size_t) STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH,
+ (const unsigned char *) sha1Digest,
+ (const size_t) STARCH2_MD_FOOTER_SHA1_LENGTH );
#endif
- /* build footer */
+/* build footer */
#ifdef DEBUG
#ifdef __cplusplus
fprintf(stderr, "\tWARNING:\nmdLength: %llu\nmd - [%s]\nsha1 - [%s]\n", static_cast<unsigned long long>( strlen(json) ), json, sha1Digest);
diff --git a/interfaces/src/data/starch/starchMetadataHelpers.c b/interfaces/src/data/starch/starchMetadataHelpers.c
index aa5617a..69b07f8 100644
--- a/interfaces/src/data/starch/starchMetadataHelpers.c
+++ b/interfaces/src/data/starch/starchMetadataHelpers.c
@@ -6,7 +6,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -56,7 +56,16 @@ namespace starch {
#endif
Metadata *
-STARCH_createMetadata(char const *chr, char const *fn, uint64_t size, LineCountType lineCount, BaseCountType totalNonUniqueBases, BaseCountType totalUniqueBases, Boolean duplicateElementExists, Boolean nestedElementExists)
+STARCH_createMetadata(char const *chr,
+ char const *fn,
+ uint64_t size,
+ LineCountType lineCount,
+ BaseCountType totalNonUniqueBases,
+ BaseCountType totalUniqueBases,
+ Boolean duplicateElementExists,
+ Boolean nestedElementExists,
+ char const *signature,
+ LineLengthType lineMaxStringLength)
{
#ifdef DEBUG
fprintf(stderr, "\n--- STARCH_createMetadata() ---\n");
@@ -64,11 +73,14 @@ STARCH_createMetadata(char const *chr, char const *fn, uint64_t size, LineCountT
Metadata *newMetadata = NULL;
size_t fnLength = 0;
size_t chrLength = 0;
+ size_t signatureLength = 0;
if (chr)
chrLength = strlen(chr);
if (fn)
fnLength = strlen(fn);
+ if (signature)
+ signatureLength = strlen(signature);
#ifdef __cplusplus
newMetadata = static_cast<Metadata *>( malloc(sizeof(Metadata)) );
@@ -78,33 +90,47 @@ STARCH_createMetadata(char const *chr, char const *fn, uint64_t size, LineCountT
if ((newMetadata != NULL) && (chr != NULL) && (fn != NULL)) {
newMetadata->chromosome = NULL;
-
#ifdef __cplusplus
newMetadata->chromosome = static_cast<char *>( malloc(chrLength + 1) );
#else
- newMetadata->chromosome = malloc(chrLength + 1);
+ newMetadata->chromosome = malloc(chrLength + 1);
#endif
-
if (!newMetadata->chromosome) {
fprintf(stderr, "ERROR: Cannot instantiate new chromosome for metadata record\n");
exit(EXIT_FAILURE);
}
strncpy(newMetadata->chromosome, chr, chrLength + 1);
- newMetadata->filename = NULL;
+ newMetadata->filename = NULL;
#ifdef __cplusplus
newMetadata->filename = static_cast<char *>( malloc(fnLength + 1) );
#else
newMetadata->filename = malloc(fnLength + 1);
#endif
-
if (!newMetadata->filename) {
fprintf(stderr, "ERROR: Cannot instantiate new filename for metadata record\n");
exit(EXIT_FAILURE);
}
strncpy(newMetadata->filename, fn, fnLength + 1);
+
+ /* we allow a NULL signature for placeholder purposes */
+ newMetadata->signature = NULL;
+ if (signature) {
+#ifdef __cplusplus
+ newMetadata->signature = static_cast<char *>( malloc(signatureLength + 1) );
+#else
+ newMetadata->signature = malloc(signatureLength + 1);
+#endif
+ if (!newMetadata->signature) {
+ fprintf(stderr, "ERROR: Cannot instantiate new signature for metadata record\n");
+ exit(EXIT_FAILURE);
+ }
+ strncpy(newMetadata->signature, signature, signatureLength + 1);
+ }
+
newMetadata->size = size;
newMetadata->lineCount = lineCount;
+ newMetadata->lineMaxStringLength = lineMaxStringLength;
newMetadata->totalNonUniqueBases = totalNonUniqueBases;
newMetadata->totalUniqueBases = totalUniqueBases;
newMetadata->duplicateElementExists = duplicateElementExists;
@@ -116,11 +142,24 @@ STARCH_createMetadata(char const *chr, char const *fn, uint64_t size, LineCountT
exit (EXIT_FAILURE);
}
+#ifdef DEBUG
+ fprintf(stderr, "\n--- STARCH_createMetadata() END ---\n");
+#endif
return newMetadata;
}
Metadata *
-STARCH_addMetadata(Metadata *md, char *chr, char *fn, uint64_t size, LineCountType lineCount, BaseCountType totalNonUniqueBases, BaseCountType totalUniqueBases, Boolean duplicateElementExists, Boolean nestedElementExists)
+STARCH_addMetadata(Metadata *md,
+ char *chr,
+ char *fn,
+ uint64_t size,
+ LineCountType lineCount,
+ BaseCountType totalNonUniqueBases,
+ BaseCountType totalUniqueBases,
+ Boolean duplicateElementExists,
+ Boolean nestedElementExists,
+ char *signature,
+ LineLengthType lineMaxStringLength)
{
#ifdef DEBUG
fprintf(stderr, "\n--- STARCH_addMetadata() ---\n");
@@ -132,7 +171,9 @@ STARCH_addMetadata(Metadata *md, char *chr, char *fn, uint64_t size, LineCountTy
totalNonUniqueBases,
totalUniqueBases,
duplicateElementExists,
- nestedElementExists);
+ nestedElementExists,
+ signature,
+ lineMaxStringLength);
if ((newMd != NULL) && (md->next == NULL))
md->next = newMd;
@@ -162,7 +203,9 @@ STARCH_copyMetadata(const Metadata *md)
md->totalNonUniqueBases,
md->totalUniqueBases,
md->duplicateElementExists,
- md->nestedElementExists);
+ md->nestedElementExists,
+ md->signature,
+ md->lineMaxStringLength);
firstRec = copy;
md = md->next;
@@ -176,7 +219,9 @@ STARCH_copyMetadata(const Metadata *md)
iter->totalNonUniqueBases,
iter->totalUniqueBases,
iter->duplicateElementExists,
- iter->nestedElementExists);
+ iter->nestedElementExists,
+ iter->signature,
+ iter->lineMaxStringLength);
}
if (!firstRec) {
@@ -188,7 +233,17 @@ STARCH_copyMetadata(const Metadata *md)
}
int
-STARCH_updateMetadataForChromosome(Metadata **md, char *chr, char *fn, uint64_t size, LineCountType lineCount, BaseCountType totalNonUniqueBases, BaseCountType totalUniqueBases, Boolean duplicateElementExists, Boolean nestedElementExists)
+STARCH_updateMetadataForChromosome(Metadata **md,
+ char *chr,
+ char *fn,
+ uint64_t size,
+ LineCountType lineCount,
+ BaseCountType totalNonUniqueBases,
+ BaseCountType totalUniqueBases,
+ Boolean duplicateElementExists,
+ Boolean nestedElementExists,
+ char *signature,
+ LineLengthType lineMaxStringLength)
{
#ifdef DEBUG
fprintf(stderr, "\n--- STARCH_updateMetadataForChromosome() ---\n");
@@ -229,12 +284,28 @@ STARCH_updateMetadataForChromosome(Metadata **md, char *chr, char *fn, uint64_t
fprintf(stderr, "ERROR: Ran out of memory for updating metadata with filename\n");
return STARCH_EXIT_FAILURE;
}
+ if (iter->signature) {
+ free(iter->signature), iter->signature = NULL;
+ }
+ if (signature) {
+#ifdef __cplusplus
+ iter->signature = static_cast<char *>( malloc(strlen(signature) + 1) );
+#else
+ iter->signature = malloc(strlen(signature) + 1);
+#endif
+ strncpy(iter->signature, signature, strlen(signature) + 1);
+ if (!iter->signature) {
+ fprintf(stderr, "ERROR: Ran out of memory for updating metadata with signature\n");
+ return STARCH_EXIT_FAILURE;
+ }
+ }
iter->size = size;
iter->lineCount = lineCount;
iter->totalNonUniqueBases = totalNonUniqueBases;
iter->totalUniqueBases = totalUniqueBases;
iter->duplicateElementExists = duplicateElementExists;
iter->nestedElementExists = nestedElementExists;
+ iter->lineMaxStringLength = lineMaxStringLength;
break;
}
}
@@ -243,7 +314,8 @@ STARCH_updateMetadataForChromosome(Metadata **md, char *chr, char *fn, uint64_t
}
int
-STARCH_listMetadata(const Metadata *md, const char *chr)
+STARCH_listMetadata(const Metadata *md,
+ const char *chr)
{
#ifdef DEBUG
fprintf(stderr, "\n--- STARCH_listMetadata() ---\n");
@@ -274,14 +346,36 @@ STARCH_listMetadata(const Metadata *md, const char *chr)
}
if (chrFound == kStarchTrue) {
- fprintf(stdout, "%-25s| %-65s\t| %-15s\t| %-20s\t| %-20s\t| %-20s\t| %-25s\t| %-25s\n", "chr", "filename", "compressedSize", "uncompressedLineCount", "totalNonUniqueBases", "totalUniqueBases", "duplicateElementExists", "nestedElementExists");
+ fprintf(stdout,
+ "%-25s|%-65s|%-15s|%-25s|%-30s|%-20s|%-20s|%-25s|%-25s|%-25s\n",
+ "chr",
+ "filename",
+ "compressedSize",
+ "uncompressedLineCount",
+ "uncompressedLineMaxStrLength",
+ "totalNonUniqueBases",
+ "totalUniqueBases",
+ "duplicateElementExists",
+ "nestedElementExists",
+ "signature");
for (iter = md; iter != NULL; iter = iter->next) {
#ifdef __cplusplus
if ( (strcmp(reinterpret_cast<const char *>( iter->chromosome ), chr) == 0) || (strcmp("all", chr) == 0) )
#else
if ( (strcmp((const char *)iter->chromosome, chr) == 0) || (strcmp("all", chr) == 0) )
#endif
- fprintf(stdout, "%-25s| %-65s\t| %-15" PRIu64 "\t| %-20" PRIu64 "\t| %-20" PRIu64 "\t| %-20" PRIu64 "\t| %-25s\t| %-25s\n", iter->chromosome, iter->filename, iter->size, iter->lineCount, iter->totalNonUniqueBases, iter->totalUniqueBases, (iter->duplicateElementExists == kStarchTrue ? t : f), (iter->nestedElementExists == kStarchTrue ? t : f));
+ fprintf(stdout,
+ "%-25s|%-65s|%-15" PRIu64 "|%-25" PRIu64 "|%-30d|%-20" PRIu64 "|%-20" PRIu64 "|%-25s|%-25s|%-25s\n",
+ iter->chromosome,
+ iter->filename,
+ iter->size,
+ iter->lineCount,
+ iter->lineMaxStringLength,
+ iter->totalNonUniqueBases,
+ iter->totalUniqueBases,
+ (iter->duplicateElementExists == kStarchTrue ? t : f),
+ (iter->nestedElementExists == kStarchTrue ? t : f),
+ (iter->signature) ? iter->signature : "NA");
}
}
@@ -309,7 +403,8 @@ STARCH_listAllChromosomes(const Metadata *md)
}
int
-STARCH_listChromosome(const Metadata *md, const char *chr)
+STARCH_listChromosome(const Metadata *md,
+ const char *chr)
{
#ifdef DEBUG
fprintf(stderr, "\n--- STARCH_listChromosome() ---\n");
@@ -349,6 +444,8 @@ STARCH_freeMetadata(Metadata **md)
free(iter->chromosome);
if (iter->filename != NULL)
free(iter->filename);
+ if (iter->signature != NULL)
+ free(iter->signature);
if (prev != NULL)
free(prev);
@@ -385,7 +482,12 @@ STARCH_deleteCompressedFiles(const Metadata *md)
}
char *
-STARCH_generateJSONMetadata(const Metadata *md, const CompressionType type, const ArchiveVersion *av, const char *cTime, const char *note, const Boolean headerFlag)
+STARCH_generateJSONMetadata(const Metadata *md,
+ const CompressionType type,
+ const ArchiveVersion *av,
+ const char *cTime,
+ const char *note,
+ const Boolean headerFlag)
{
#ifdef DEBUG
fprintf(stderr, "\n--- STARCH_generateJSONMetadata() ---\n");
@@ -403,11 +505,13 @@ STARCH_generateJSONMetadata(const Metadata *md, const CompressionType type, cons
json_t *streamFilename = NULL;
json_t *streamSize = NULL;
json_t *streamLineCount = NULL;
+ json_t *streamLineMaxStringLength = NULL;
json_t *streamTotalNonUniqueBases = NULL;
json_t *streamTotalUniqueBases = NULL;
json_t *streamCustomHeaderFlag = NULL;
json_t *streamDuplicateElementExistsFlag = NULL;
json_t *streamNestedElementExistsFlag = NULL;
+ json_t *streamSignature = NULL;
json_t *streamArchive = NULL;
json_t *streamArchiveType = NULL;
json_t *streamArchiveNote = NULL;
@@ -418,6 +522,7 @@ STARCH_generateJSONMetadata(const Metadata *md, const CompressionType type, cons
json_t *streamArchiveVersionRevision = NULL;
char *recordFilenameCopy = NULL;
char *recordChromosome = NULL;
+ char *recordSignature = NULL;
char *recordToken = NULL;
char *recordSize = NULL;
char *creationTimestamp = NULL;
@@ -425,6 +530,7 @@ STARCH_generateJSONMetadata(const Metadata *md, const CompressionType type, cons
size_t creationTimestampLength = STARCH_CREATION_TIMESTAMP_LENGTH;
uint64_t filenameSize = 0;
LineCountType filenameLineCount = 0;
+ LineLengthType filenameLineMaxStringLength = 0UL;
BaseCountType totalNonUniqueBases = 0;
BaseCountType totalUniqueBases = 0;
time_t creationTime;
@@ -611,6 +717,27 @@ STARCH_generateJSONMetadata(const Metadata *md, const CompressionType type, cons
json_object_set_new(stream, STARCH_METADATA_STREAM_NESTEDELEMENTEXISTS_KEY, streamNestedElementExistsFlag);
}
+ /* 2.2+ archive */
+ if ((json_integer_value(streamArchiveVersionMajor) > 2) || ((json_integer_value(streamArchiveVersionMajor) == 2) && (json_integer_value(streamArchiveVersionMinor) >= 2))) {
+ if ((iter->signature) && (strlen(iter->signature) > 0)) {
+ /* data integrity signature */
+ recordSignature = STARCH_strndup(iter->signature, strlen(iter->signature) + 1);
+ streamSignature = json_string(recordSignature);
+ json_object_set_new(stream, STARCH_METADATA_STREAM_SIGNATURE_KEY, streamSignature);
+ free(recordSignature);
+ }
+ if (iter->lineMaxStringLength > 0) {
+ /* maximum string length */
+ filenameLineMaxStringLength = iter->lineMaxStringLength;
+#ifdef __cplusplus
+ streamLineMaxStringLength = json_integer(static_cast<json_int_t>(filenameLineMaxStringLength));
+#else
+ streamLineMaxStringLength = json_integer((json_int_t)filenameLineMaxStringLength);
+#endif
+ json_object_set_new(stream, STARCH_METADATA_STREAM_LINEMAXSTRINGLENGTH_KEY, streamLineMaxStringLength);
+ }
+ }
+
json_array_append_new(streams, stream);
}
@@ -629,7 +756,15 @@ STARCH_generateJSONMetadata(const Metadata *md, const CompressionType type, cons
}
int
-STARCH_listJSONMetadata(FILE *out, FILE *err, const Metadata *md, const CompressionType type, const ArchiveVersion *av, const char *cTime, const char *note, const Boolean headerFlag, const Boolean showNewlineFlag)
+STARCH_listJSONMetadata(FILE *out,
+ FILE *err,
+ const Metadata *md,
+ const CompressionType type,
+ const ArchiveVersion *av,
+ const char *cTime,
+ const char *note,
+ const Boolean headerFlag,
+ const Boolean showNewlineFlag)
{
#ifdef DEBUG
fprintf(stderr, "\n--- STARCH_listJSONMetadata() ---\n");
@@ -658,7 +793,11 @@ STARCH_listJSONMetadata(FILE *out, FILE *err, const Metadata *md, const Compress
}
int
-STARCH_writeJSONMetadata(const Metadata *md, char **buf, CompressionType *type, const Boolean headerFlag, const char *note)
+STARCH_writeJSONMetadata(const Metadata *md,
+ char **buf,
+ CompressionType *type,
+ const Boolean headerFlag,
+ const char *note)
{
#ifdef DEBUG
fprintf(stderr, "\n--- STARCH_writeJSONMetadata() ---\n");
@@ -708,7 +847,18 @@ STARCH_writeJSONMetadata(const Metadata *md, char **buf, CompressionType *type,
}
int
-STARCH_readJSONMetadata(json_t **metadataJSON, FILE **fp, const char *fn, Metadata **rec, CompressionType *type, ArchiveVersion **version, char **cTime, char **note, uint64_t *mdOffset, Boolean *headerFlag, const Boolean suppressErrorMsgs, const Boolean preserveJSONRef)
+STARCH_readJSONMetadata(json_t **metadataJSON,
+ FILE **fp,
+ const char *fn,
+ Metadata **rec,
+ CompressionType *type,
+ ArchiveVersion **version,
+ char **cTime,
+ char **note,
+ uint64_t *mdOffset,
+ Boolean *headerFlag,
+ const Boolean suppressErrorMsgs,
+ const Boolean preserveJSONRef)
{
#ifdef DEBUG
fprintf(stderr, "\n--- STARCH_readJSONMetadata() ---\n");
@@ -732,8 +882,10 @@ STARCH_readJSONMetadata(json_t **metadataJSON, FILE **fp, const char *fn, Metada
json_t *stream = NULL;
json_t *streamChromosome = NULL;
json_t *streamFilename = NULL;
+ json_t *streamSignature = NULL;
json_t *streamSize = NULL;
json_t *streamLineCount = NULL;
+ json_t *streamLineMaxStringLength = NULL;
json_t *streamTotalNonUniqueBases = NULL;
json_t *streamTotalUniqueBases = NULL;
json_t *streamDuplicateElementExistsFlag = NULL;
@@ -744,6 +896,7 @@ STARCH_readJSONMetadata(json_t **metadataJSON, FILE **fp, const char *fn, Metada
char *streamFn = NULL;
char *streamCTime = NULL;
char *streamNote = NULL;
+ char *streamSig = NULL;
uint64_t streamSizeValue = 0;
char *testMagicPrecursor = NULL;
json_error_t jsonParseError;
@@ -752,6 +905,7 @@ STARCH_readJSONMetadata(json_t **metadataJSON, FILE **fp, const char *fn, Metada
const char *jsonObjAvKey = NULL;
json_t *jsonObjAvValue = NULL;
LineCountType streamLineCountValue = STARCH_DEFAULT_LINE_COUNT;
+ LineLengthType streamLineMaxStringLengthValue = STARCH_DEFAULT_LINE_STRING_LENGTH;
BaseCountType streamTotalNonUniqueBasesValue = STARCH_DEFAULT_NON_UNIQUE_BASE_COUNT;
BaseCountType streamTotalUniqueBasesValue = STARCH_DEFAULT_UNIQUE_BASE_COUNT;
Boolean streamDuplicateElementExistsValue = STARCH_DEFAULT_DUPLICATE_ELEMENT_FLAG_VALUE;
@@ -1275,6 +1429,17 @@ STARCH_readJSONMetadata(json_t **metadataJSON, FILE **fp, const char *fn, Metada
return STARCH_FATAL_ERROR;
}
+ #ifdef __cplusplus
+ streamSig = static_cast<char *>( malloc( STARCH_STREAM_METADATA_MAX_LENGTH + 1 ) );
+#else
+ streamSig = malloc( STARCH_STREAM_METADATA_MAX_LENGTH + 1 );
+#endif
+ if (!streamSig) {
+ if (suppressErrorMsgs == kStarchFalse)
+ fprintf(stderr, "ERROR: Could not instantiate memory for stream signature string.\n");
+ return STARCH_FATAL_ERROR;
+ }
+
for (streamIdx = 0; streamIdx < json_array_size(streams); streamIdx++) {
stream = json_array_get(streams, streamIdx);
if (!stream) {
@@ -1379,7 +1544,7 @@ STARCH_readJSONMetadata(json_t **metadataJSON, FILE **fp, const char *fn, Metada
#else
streamTotalUniqueBasesValue = (BaseCountType) json_integer_value(streamTotalUniqueBases);
#endif
- }
+ }
streamDuplicateElementExistsFlag = json_object_get(stream, STARCH_METADATA_STREAM_DUPLICATEELEMENTEXISTS_KEY);
if (!streamDuplicateElementExistsFlag) {
@@ -1404,7 +1569,7 @@ STARCH_readJSONMetadata(json_t **metadataJSON, FILE **fp, const char *fn, Metada
#else
streamDuplicateElementExistsValue = (Boolean) json_is_true(streamDuplicateElementExistsFlag);
#endif
- }
+ }
streamNestedElementExistsFlag = json_object_get(stream, STARCH_METADATA_STREAM_NESTEDELEMENTEXISTS_KEY);
if (!streamNestedElementExistsFlag) {
@@ -1429,17 +1594,53 @@ STARCH_readJSONMetadata(json_t **metadataJSON, FILE **fp, const char *fn, Metada
#else
streamNestedElementExistsValue = (Boolean) json_is_true(streamNestedElementExistsFlag);
#endif
- }
+ }
+
+ /* v2.2+ */
+ if (((*version)->major > 2) || (((*version)->major == 2) && ((*version)->minor >= 2))) {
+ /* data integrity signature */
+ streamSignature = json_object_get(stream, STARCH_METADATA_STREAM_SIGNATURE_KEY);
+ if (streamSignature) {
+ strncpy(streamSig, json_string_value(streamSignature), strlen(json_string_value(streamSignature)) + 1);
+ }
+ /* maximum string length */
+ streamLineMaxStringLength = json_object_get(stream, STARCH_METADATA_STREAM_LINEMAXSTRINGLENGTH_KEY);
+#ifdef __cplusplus
+ streamLineMaxStringLengthValue = static_cast<LineLengthType>( json_integer_value(streamLineMaxStringLength) );
+#else
+ streamLineMaxStringLengthValue = (LineLengthType) json_integer_value(streamLineMaxStringLength);
+#endif
+ }
+
strncpy(streamChr, json_string_value(streamChromosome), strlen(json_string_value(streamChromosome)) + 1);
strncpy(streamFn, json_string_value(streamFilename), strlen(json_string_value(streamFilename)) + 1);
if (streamIdx == 0) {
- *rec = STARCH_createMetadata(streamChr, streamFn, streamSizeValue, streamLineCountValue, streamTotalNonUniqueBasesValue, streamTotalUniqueBasesValue, streamDuplicateElementExistsValue, streamNestedElementExistsValue);
+ *rec = STARCH_createMetadata(streamChr,
+ streamFn,
+ streamSizeValue,
+ streamLineCountValue,
+ streamTotalNonUniqueBasesValue,
+ streamTotalUniqueBasesValue,
+ streamDuplicateElementExistsValue,
+ streamNestedElementExistsValue,
+ streamSig,
+ streamLineMaxStringLengthValue);
firstRec = *rec;
}
else
- *rec = STARCH_addMetadata(*rec, streamChr, streamFn, streamSizeValue, streamLineCountValue, streamTotalNonUniqueBasesValue, streamTotalUniqueBasesValue, streamDuplicateElementExistsValue, streamNestedElementExistsValue);
+ *rec = STARCH_addMetadata(*rec,
+ streamChr,
+ streamFn,
+ streamSizeValue,
+ streamLineCountValue,
+ streamTotalNonUniqueBasesValue,
+ streamTotalUniqueBasesValue,
+ streamDuplicateElementExistsValue,
+ streamNestedElementExistsValue,
+ streamSig,
+ streamLineMaxStringLengthValue);
}
/* reset Metadata record pointer to first record */
@@ -1454,7 +1655,9 @@ STARCH_readJSONMetadata(json_t **metadataJSON, FILE **fp, const char *fn, Metada
if (streamChr)
free(streamChr);
if (streamFn)
- free(streamFn);
+ free(streamFn);
+ if (streamSig)
+ free(streamSig);
if ((mdJSON != NULL) && (preserveJSONRef == kStarchFalse)) {
json_decref(mdJSON);
mdJSON = NULL;
@@ -1471,7 +1674,8 @@ STARCH_readJSONMetadata(json_t **metadataJSON, FILE **fp, const char *fn, Metada
}
int
-STARCH_mergeMetadataWithCompressedFiles(const Metadata *md, char *mdHeader)
+STARCH_mergeMetadataWithCompressedFiles(const Metadata *md,
+ char *mdHeader)
{
#ifdef DEBUG
fprintf(stderr, "\n--- STARCH_mergeMetadataWithCompressedFiles() ---\n");
@@ -1543,7 +1747,13 @@ STARCH_copyArchiveVersion(const ArchiveVersion *oav)
}
int
-STARCH_readLegacyMetadata(const char *buf, Metadata **rec, CompressionType *type, ArchiveVersion **version, uint64_t *mdOffset, Boolean *headerFlag, const Boolean suppressErrorMsgs)
+STARCH_readLegacyMetadata(const char *buf,
+ Metadata **rec,
+ CompressionType *type,
+ ArchiveVersion **version,
+ uint64_t *mdOffset,
+ Boolean *headerFlag,
+ const Boolean suppressErrorMsgs)
{
#ifdef DEBUG
fprintf(stderr, "\n--- STARCH_readLegacyMetadata() ---\n");
@@ -1675,11 +1885,30 @@ STARCH_readLegacyMetadata(const char *buf, Metadata **rec, CompressionType *type
if (tokBufIdx != 0) {
/* put record into metadata */
if (recIdx == 1) {
- *rec = STARCH_createMetadata(recChromosome, recFilename, recFileSize, recLineCountValue, recNonUniqueBaseCountValue, recUniqueBaseCountValue, recDuplicateElementExistsFlagValue, recNestedElementExistsFlagValue);
+ *rec = STARCH_createMetadata(recChromosome,
+ recFilename,
+ recFileSize,
+ recLineCountValue,
+ recNonUniqueBaseCountValue,
+ recUniqueBaseCountValue,
+ recDuplicateElementExistsFlagValue,
+ recNestedElementExistsFlagValue,
+ NULL,
+ STARCH_DEFAULT_LINE_STRING_LENGTH);
firstRec = *rec;
}
else
- *rec = STARCH_addMetadata(*rec, recChromosome, recFilename, recFileSize, recLineCountValue, recNonUniqueBaseCountValue, recUniqueBaseCountValue, recDuplicateElementExistsFlagValue, recNestedElementExistsFlagValue);
+ *rec = STARCH_addMetadata(*rec,
+ recChromosome,
+ recFilename,
+ recFileSize,
+ recLineCountValue,
+ recNonUniqueBaseCountValue,
+ recUniqueBaseCountValue,
+ recDuplicateElementExistsFlagValue,
+ recNestedElementExistsFlagValue,
+ NULL,
+ STARCH_DEFAULT_LINE_STRING_LENGTH);
/* cleanup */
if (recFilename)
@@ -1718,7 +1947,9 @@ STARCH_readLegacyMetadata(const char *buf, Metadata **rec, CompressionType *type
}
char *
-STARCH_strnstr(const char *haystack, const char *needle, size_t haystackLen)
+STARCH_strnstr(const char *haystack,
+ const char *needle,
+ size_t haystackLen)
{
#ifdef DEBUG
fprintf(stderr, "\n--- STARCH_strnstr() ---\n");
@@ -1754,7 +1985,8 @@ STARCH_strnstr(const char *haystack, const char *needle, size_t haystackLen)
}
int
-STARCH_chromosomeInMetadataRecords(const Metadata *md, const char *chr) {
+STARCH_chromosomeInMetadataRecords(const Metadata *md,
+ const char *chr) {
#ifdef DEBUG
fprintf(stderr, "\n--- STARCH_chromosomeInMetadataRecords() ---\n");
#endif
@@ -1772,6 +2004,26 @@ STARCH_chromosomeInMetadataRecords(const Metadata *md, const char *chr) {
return STARCH_EXIT_FAILURE;
}
+int
+STARCH_chromosomePositionedBeforeExistingMetadataRecord(const Metadata *md,
+ const char *chr) {
+#ifdef DEBUG
+ fprintf(stderr, "\n--- STARCH_chromosomePositionedBeforeExistingMetadataRecord() ---\n");
+#endif
+ const Metadata *iter;
+
+ if (!md) {
+ fprintf(stderr, "ERROR: Could not list chromosomes (metadata structure is empty)\n");
+ return STARCH_EXIT_FAILURE;
+ }
+
+ for (iter = md; iter != NULL; iter = iter->next)
+ if (strcmp(chr, iter->chromosome) < 0)
+ return STARCH_EXIT_SUCCESS;
+
+ return STARCH_EXIT_FAILURE;
+}
+
#ifdef __cplusplus
} // namespace starch
#endif
diff --git a/interfaces/src/data/starch/unstarchHelpers.c b/interfaces/src/data/starch/unstarchHelpers.c
index 81cf40d..255bb5d 100644
--- a/interfaces/src/data/starch/unstarchHelpers.c
+++ b/interfaces/src/data/starch/unstarchHelpers.c
@@ -6,7 +6,7 @@
//
// BEDOPS
-// Copyright (C) 2011-2016 Shane Neph, Scott Kuehn and Alex Reynolds
+// Copyright (C) 2011-2017 Shane Neph, Scott Kuehn and Alex Reynolds
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -41,6 +41,8 @@
#include <zlib.h>
+#include "data/starch/starchSha1Digest.h"
+#include "data/starch/starchBase64Coding.h"
#include "data/starch/starchFileHelpers.h"
#include "data/starch/starchHelpers.h"
#include "data/starch/unstarchHelpers.h"
@@ -298,7 +300,7 @@ UNSTARCH_extractDataWithBzip2(FILE **inFp, FILE *outFp, const char *whichChr, co
#else
if (STARCH_fseeko(*inFp, (off_t) (cumulativeSize + mdOffset), SEEK_SET) != 0) {
#endif
- fprintf(stderr, "ERROR: Could not seek data in archve\n");
+ fprintf(stderr, "ERROR: Could not seek data in archive\n");
return UNSTARCH_FATAL_ERROR;
}
cumulativeSize += size;
@@ -1699,6 +1701,55 @@ UNSTARCH_printLineCountForAllChromosomes(const Metadata *md)
fprintf(stdout, "%" PRIu64 "\n", totalLineCount);
}
+LineLengthType
+UNSTARCH_lineMaxStringLengthForChromosome(const Metadata *md, const char *chr)
+{
+#ifdef DEBUG
+ fprintf(stderr, "\n--- UNSTARCH_lineMaxStringLengthForChromosome() ---\n");
+#endif
+ const Metadata *iter;
+
+ for (iter = md; iter != NULL; iter = iter->next) {
+ if (strcmp(chr, iter->chromosome) == 0) {
+ return iter->lineMaxStringLength;
+ }
+ }
+
+#ifdef __cplusplus
+ return static_cast<LineLengthType>( STARCH_DEFAULT_LINE_STRING_LENGTH );
+#else
+ return (LineLengthType) STARCH_DEFAULT_LINE_STRING_LENGTH;
+#endif
+}
+
+void
+UNSTARCH_printLineMaxStringLengthForChromosome(const Metadata *md, const char *chr)
+{
+#ifdef DEBUG
+ fprintf(stderr, "\n--- UNSTARCH_printLineMaxStringLengthForChromosome() ---\n");
+#endif
+
+ if (strcmp(chr, "all") == 0)
+ UNSTARCH_printLineMaxStringLengthForAllChromosomes(md);
+ else
+ fprintf(stdout, "%d\n", UNSTARCH_lineMaxStringLengthForChromosome(md, chr));
+}
+
+void
+UNSTARCH_printLineMaxStringLengthForAllChromosomes(const Metadata *md)
+{
+#ifdef DEBUG
+ fprintf(stderr, "\n--- UNSTARCH_printLineMaxStringLengthForAllChromosomes() ---\n");
+#endif
+ const Metadata *iter;
+ LineLengthType lineMaxStringLength = STARCH_DEFAULT_LINE_STRING_LENGTH;
+
+ for (iter = md; iter != NULL; iter = iter->next)
+ lineMaxStringLength = (lineMaxStringLength >= iter->lineMaxStringLength) ? lineMaxStringLength : iter->lineMaxStringLength;
+
+ fprintf(stdout, "%d\n", lineMaxStringLength);
+}
+
BaseCountType
UNSTARCH_nonUniqueBaseCountForChromosome(const Metadata *md, const char *chr)
{
@@ -1989,6 +2040,385 @@ UNSTARCH_printNestedElementExistsIntegersForAllChromosomes(const Metadata *md)
#endif
}
+void
+UNSTARCH_printSignature(const Metadata *md, const char *chr, const unsigned char *mdSha1Buffer)
+{
+#ifdef DEBUG
+ fprintf(stderr, "\n--- UNSTARCH_printSignature() ---\n");
+#endif
+ char *signature = NULL;
+
+ if (strcmp(chr, "all") == 0) {
+ UNSTARCH_printAllSignatures(md, mdSha1Buffer);
+ }
+ else {
+ signature = UNSTARCH_signatureForChromosome(md, chr);
+ if (signature)
+ fprintf(stdout, "%s\n", signature);
+ }
+}
+
+char *
+UNSTARCH_signatureForChromosome(const Metadata *md, const char *chr)
+{
+#ifdef DEBUG
+ fprintf(stderr, "\n--- UNSTARCH_signatureForChromosome() ---\n");
+#endif
+ const Metadata *iter;
+
+ for (iter = md; iter != NULL; iter = iter->next) {
+ if ((strcmp(chr, iter->chromosome) == 0) && (iter->signature) && (strlen(iter->signature) > 0))
+ return iter->signature;
+ }
+
+ return NULL;
+}
+
+void
+UNSTARCH_printAllSignatures(const Metadata *md, const unsigned char *mdSha1Buffer)
+{
+#ifdef DEBUG
+ fprintf(stderr, "\n--- UNSTARCH_printAllSignatures() ---\n");
+#endif
+ UNSTARCH_printMetadataSignature(mdSha1Buffer);
+ UNSTARCH_printAllChromosomeSignatures(md);
+}
+
+void
+UNSTARCH_printMetadataSignature(const unsigned char *mdSha1Buffer)
+{
+#ifdef DEBUG
+ fprintf(stderr, "\n--- UNSTARCH_printMetadataSignature() ---\n");
+#endif
+ size_t mdSha1BufferLength = STARCH2_MD_FOOTER_SHA1_LENGTH;
+ char *jsonBase64String = NULL;
+
+#ifdef __cplusplus
+ STARCH_encodeBase64(&jsonBase64String,
+ static_cast<const size_t>( STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH ),
+ const_cast<const unsigned char *>( mdSha1Buffer ),
+ static_cast<const size_t>( mdSha1BufferLength ));
+#else
+ STARCH_encodeBase64(&jsonBase64String,
+ (const size_t) STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH,
+ (const unsigned char *) mdSha1Buffer,
+ (const size_t) mdSha1BufferLength);
+#endif
+
+ if (!jsonBase64String) {
+ fprintf(stderr, "ERROR: Could not allocate space for Base64-encoded metadata string representation\n");
+ exit(-1);
+ }
+ fprintf(stdout, "metadata\t%s\n", jsonBase64String);
+ free(jsonBase64String), jsonBase64String = NULL;
+}
+
+void
+UNSTARCH_printAllChromosomeSignatures(const Metadata *md)
+{
+#ifdef DEBUG
+ fprintf(stderr, "\n--- UNSTARCH_printAllChromosomeSignatures() ---\n");
+#endif
+ const Metadata *iter;
+
+ for (iter = md; iter != NULL; iter = iter->next) {
+ fprintf(stdout, "%s\t%s\n", iter->chromosome, strlen(iter->signature) > 0 ? iter->signature : "NA");
+ }
+}
+
+Boolean
+UNSTARCH_verifySignature(FILE **inFp, const Metadata *md, const uint64_t mdOffset, const char *chr, CompressionType compType)
+{
+#ifdef DEBUG
+ fprintf(stderr, "\n--- UNSTARCH_verifySignature() ---\n");
+#endif
+ char *expectedSignature = NULL;
+ char *observedSignature = NULL;
+ Boolean signaturesVerifiedFlag = kStarchFalse;
+
+ if (strcmp(chr, "all") == 0) {
+ signaturesVerifiedFlag = UNSTARCH_verifyAllSignatures(inFp, md, mdOffset, compType);
+ }
+ else {
+ expectedSignature = UNSTARCH_signatureForChromosome(md, chr);
+ if (!expectedSignature) {
+ fprintf(stderr, "ERROR: Could not locate signature in metadata for specified chromosome name for purposes of verification [%s]\n", chr);
+ signaturesVerifiedFlag = kStarchFalse;
+ }
+ else {
+ observedSignature = UNSTARCH_observedSignatureForChromosome(inFp, md, mdOffset, chr, compType);
+ if (!observedSignature) {
+ signaturesVerifiedFlag = kStarchFalse;
+ }
+ }
+ if ((expectedSignature) && (observedSignature)) {
+ if (strcmp(observedSignature, expectedSignature) != 0) {
+ fprintf(stderr, "ERROR: Specified chromosome record may be corrupt -- observed and expected signatures do not match for chromosome [%s]\n", chr);
+ signaturesVerifiedFlag = kStarchFalse;
+ }
+ else {
+ fprintf(stderr, "Expected and observed data integrity signatures match for chromosome [%s]\n", chr);
+ signaturesVerifiedFlag = kStarchTrue;
+ }
+ }
+ }
+ if (observedSignature) { free(observedSignature), observedSignature = NULL; }
+ return signaturesVerifiedFlag;
+}
+
+char *
+UNSTARCH_observedSignatureForChromosome(FILE **inFp, const Metadata *md, const uint64_t mdOffset, const char *chr, CompressionType compType)
+{
+#ifdef DEBUG
+ fprintf(stderr, "\n--- UNSTARCH_observedSignatureForChromosome() ---\n");
+#endif
+
+ /*
+ 1) Open file pointer to specified offset
+ 2) Extract transformed data from compressed chromosome stream
+ 3) Run block SHA-1 function on byte stream until end-of-stream
+ 4) Return Base64 encoding of SHA-1 signature
+ */
+
+ const Metadata *iter = NULL;
+ char *currentChromosome = NULL;
+ uint64_t size = 0;
+ uint64_t cumulativeSize = 0;
+
+ unsigned char sha1Digest[STARCH2_MD_FOOTER_SHA1_LENGTH] = {0};
+ char *base64EncodedSha1Digest = NULL;
+ struct sha1_ctx perChromosomeHashCtx;
+
+ for (iter = md; iter != NULL; iter = iter->next) {
+ currentChromosome = iter->chromosome;
+ size = iter->size;
+#ifdef __cplusplus
+ if (STARCH_fseeko(*inFp, static_cast<off_t>( cumulativeSize + mdOffset ), SEEK_SET) != 0) {
+ fprintf(stderr, "ERROR: Could not seek data in archive\n");
+ return NULL;
+ }
+#else
+ if (STARCH_fseeko(*inFp, (off_t) (cumulativeSize + mdOffset), SEEK_SET) != 0) {
+ fprintf(stderr, "ERROR: Could not seek data in archive\n");
+ return NULL;
+ }
+#endif
+ cumulativeSize += size;
+ if (strcmp(chr, currentChromosome) == 0) {
+
+ // depending on archive compression type (bzip2 or gzip) we set up
+ // the machinery to extract a stream of transformed data out of the
+ // compressed bytes. we run our sha1_process_bytes() call on the
+ // transformed data
+
+ // initialize hash context
+ sha1_init_ctx (&perChromosomeHashCtx);
+
+ switch (compType) {
+ case kBzip2: {
+ BZFILE *bzFp = NULL;
+ int bzError = 0;
+ unsigned char *bzOutput = NULL;
+ size_t bzOutputLength = UNSTARCH_COMPRESSED_BUFFER_MAX_LENGTH;
+ bzFp = BZ2_bzReadOpen( &bzError, *inFp, 0, 0, NULL, 0 ); /* http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html#bzcompress-init */
+ if (bzError != BZ_OK) {
+ BZ2_bzReadClose( &bzError, bzFp );
+ fprintf(stderr, "ERROR: Bzip2 data stream could not be opened\n");
+ return NULL;
+ }
+#ifdef __cplusplus
+ bzOutput = static_cast<unsigned char *>( malloc(bzOutputLength) );
+#else
+ bzOutput = malloc(bzOutputLength);
+#endif
+ do {
+ UNSTARCH_bzReadLine(bzFp, &bzOutput);
+ if (bzOutput) {
+ /*
+ The output of UNSTARCH_bzReadLine strips the newline character, because
+ the transformation tokens do not need a newline when being turned back
+ into raw BED.
+
+ When the raw BED was originally turned into tranform tokens, the so-called
+ "transformation" buffer contained these newline characters. So we put them
+ back in the bzOutput buffer and add one byte to the string length.
+
+ This modified buffer is what goes into sha1_process_bytes().
+ */
+#ifdef __cplusplus
+ size_t len = strlen(reinterpret_cast<const char *>( bzOutput ));
+#else
+ size_t len = strlen((const char *)bzOutput);
+#endif
+ bzOutput[len] = '\n';
+ bzOutput[++len] = '\0';
+ sha1_process_bytes( bzOutput, len, &perChromosomeHashCtx );
+ }
+ } while (bzOutput != NULL);
+ /* cleanup */
+ if (bzOutput)
+ free(bzOutput);
+ BZ2_bzReadClose(&bzError, bzFp);
+ break;
+ }
+ case kGzip: {
+ z_stream zStream;
+ unsigned int zHave, zOutBufIdx;
+ size_t zBufIdx, zBufOffset;
+ int zError;
+ unsigned char *zRemainderBuf = NULL;
+ unsigned char zInBuf[STARCH_Z_CHUNK];
+ unsigned char zOutBuf[STARCH_Z_CHUNK];
+ unsigned char zLineBuf[STARCH_Z_CHUNK];
+
+ zStream.zalloc = Z_NULL;
+ zStream.zfree = Z_NULL;
+ zStream.opaque = Z_NULL;
+ zStream.avail_in = 0;
+ zStream.next_in = Z_NULL;
+
+ zError = inflateInit2(&zStream, (15+32)); /* cf. http://www.zlib.net/manual.html */
+ if (zError != Z_OK) {
+ fprintf(stderr, "ERROR: Could not initialize z-stream\n");
+ return NULL;
+ }
+
+#ifdef __cplusplus
+ zRemainderBuf = static_cast<unsigned char *>( malloc(1) );
+#else
+ zRemainderBuf = (unsigned char *) malloc(1);
+#endif
+ *zRemainderBuf = '\0';
+
+ do {
+#ifdef __cplusplus
+ zStream.avail_in = static_cast<unsigned int>( fread(zInBuf, 1, STARCH_Z_CHUNK, *inFp) );
+#else
+ zStream.avail_in = (unsigned int) fread(zInBuf, 1, STARCH_Z_CHUNK, *inFp);
+#endif
+ if (zStream.avail_in == 0)
+ break;
+ zStream.next_in = zInBuf;
+ do {
+ zStream.avail_out = STARCH_Z_CHUNK;
+ zStream.next_out = zOutBuf;
+ zError = inflate(&zStream, Z_NO_FLUSH);
+ switch (zError) {
+ case Z_NEED_DICT: { fprintf(stderr, "ERROR: Z-stream needs dictionary\n"); return NULL; }
+ case Z_DATA_ERROR: { fprintf(stderr, "ERROR: Z-stream suffered data error\n"); return NULL; }
+ case Z_MEM_ERROR: { fprintf(stderr, "ERROR: Z-stream suffered memory error\n"); return NULL; }
+ };
+ zHave = STARCH_Z_CHUNK - zStream.avail_out;
+ zOutBuf[zHave] = '\0';
+ /* copy remainder buffer onto line buffer, if not NULL */
+#ifdef __cplusplus
+ if (zRemainderBuf) {
+ strncpy(reinterpret_cast<char *>( zLineBuf ), reinterpret_cast<const char *>( zRemainderBuf ), strlen(reinterpret_cast<const char *>( zRemainderBuf )));
+ zBufOffset = strlen(reinterpret_cast<const char *>( zRemainderBuf ));
+ }
+#else
+ if (zRemainderBuf) {
+ strncpy((char *) zLineBuf, (const char *) zRemainderBuf, strlen((const char *) zRemainderBuf));
+ zBufOffset = strlen((const char *) zRemainderBuf);
+ }
+#endif
+ else {
+ zBufOffset = 0;
+ }
+ /* read through zOutBuf for newlines */
+ for (zBufIdx = zBufOffset, zOutBufIdx = 0; zOutBufIdx < zHave; zBufIdx++, zOutBufIdx++) {
+ zLineBuf[zBufIdx] = zOutBuf[zOutBufIdx];
+ if (zLineBuf[zBufIdx] == '\n') {
+ zLineBuf[zBufIdx + 1] = '\0';
+ sha1_process_bytes( zLineBuf, zBufIdx + 1, &perChromosomeHashCtx );
+#ifdef __cplusplus
+ zBufIdx = static_cast<size_t>( -1 );
+#else
+ zBufIdx = (size_t) -1;
+#endif
+ }
+ }
+#ifdef __cplusplus
+ if (strlen(reinterpret_cast<const char *>( zLineBuf )) > 0) {
+ if (strlen(reinterpret_cast<const char *>( zLineBuf )) > strlen(reinterpret_cast<const char *>( zRemainderBuf ))) {
+ free(zRemainderBuf);
+ zRemainderBuf = reinterpret_cast<unsigned char *>( malloc(strlen(reinterpret_cast<const char *>( zLineBuf )) * 2) );
+ }
+ strncpy(reinterpret_cast<char *>( zRemainderBuf ), reinterpret_cast<const char *>( zLineBuf ), zBufIdx);
+ zRemainderBuf[zBufIdx] = '\0';
+ }
+#else
+ if (strlen((const char *) zLineBuf) > 0) {
+ if (strlen((const char *) zLineBuf) > strlen((const char *) zRemainderBuf)) {
+ free(zRemainderBuf);
+ zRemainderBuf = (unsigned char *) malloc(strlen((const char *) zLineBuf) * 2);
+ }
+ strncpy((char *) zRemainderBuf, (const char *) zLineBuf, zBufIdx);
+ zRemainderBuf[zBufIdx] = '\0';
+ }
+#endif
+ } while (zStream.avail_out == 0);
+ } while (zError != Z_STREAM_END);
+
+ /* cleanup */
+ if (zRemainderBuf) {
+ free(zRemainderBuf);
+ zRemainderBuf = NULL;
+ }
+ /* close gzip stream */
+ zError = inflateEnd(&zStream);
+ if (zError != Z_OK) {
+ fprintf(stderr, "ERROR: Could not close z-stream (%d)\n", zError);
+ return NULL;
+ }
+ break;
+ }
+ case kUndefined: {
+ fprintf(stderr, "ERROR: Archive compression type is undefined\n");
+ return NULL;
+ }
+ }
+
+ sha1_finish_ctx (&perChromosomeHashCtx, sha1Digest);
+#ifdef __cplusplus
+ STARCH_encodeBase64(&base64EncodedSha1Digest,
+ static_cast<const size_t>( STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH ),
+ const_cast<const unsigned char *>( sha1Digest ),
+ static_cast<const size_t>( STARCH2_MD_FOOTER_SHA1_LENGTH ));
+#else
+ STARCH_encodeBase64(&base64EncodedSha1Digest,
+ (const size_t) STARCH2_MD_FOOTER_BASE64_ENCODED_SHA1_LENGTH,
+ (const unsigned char *) sha1Digest,
+ (const size_t) STARCH2_MD_FOOTER_SHA1_LENGTH);
+#endif
+ return base64EncodedSha1Digest;
+ }
+ }
+ fprintf(stderr, "ERROR: Leaving UNSTARCH_observedSignatureForChromosome() without having processed chromosome [%s]\n", chr);
+ return NULL;
+}
+
+Boolean
+UNSTARCH_verifyAllSignatures(FILE **inFp, const Metadata *md, const uint64_t mdOffset, CompressionType compType)
+{
+#ifdef DEBUG
+ fprintf(stderr, "\n--- UNSTARCH_verifyAllSignatures() ---\n");
+#endif
+
+ const Metadata *iter = NULL;
+ Boolean perChromosomeSignatureVerifiedFlag = kStarchFalse;
+ Boolean allSignaturesVerifiedFlag = kStarchTrue;
+
+ for (iter = md; iter != NULL; iter = iter->next) {
+ perChromosomeSignatureVerifiedFlag = UNSTARCH_verifySignature(inFp, md, mdOffset, iter->chromosome, compType);
+ if (!perChromosomeSignatureVerifiedFlag) {
+ allSignaturesVerifiedFlag = kStarchFalse;
+ }
+ }
+
+ return allSignaturesVerifiedFlag;
+}
+
const char *
UNSTARCH_booleanToString(const Boolean val)
{
diff --git a/packaging/deb/Dockerfile b/packaging/deb/Dockerfile
index c97d630..d42f210 100644
--- a/packaging/deb/Dockerfile
+++ b/packaging/deb/Dockerfile
@@ -10,16 +10,16 @@ RUN apt-get clean
ADD . /bedops
# populate base debian package tree
-RUN mkdir /bedops_2.4.20
-WORKDIR /bedops_2.4.20
+RUN mkdir /bedops_2.4.26
+WORKDIR /bedops_2.4.26
RUN mkdir -p DEBIAN usr/bin usr/share/doc/bedops
RUN cp /bedops/LICENSE /bedops/README.md usr/share/doc/bedops
RUN cp /bedops/packaging/deb/control DEBIAN
# build and install bedops into debian package tree
WORKDIR /bedops
-RUN make -j `nproc` && make install BINDIR=/bedops_2.4.20/usr/bin
+RUN make -j `nproc` && make install BINDIR=/bedops_2.4.26/usr/bin
WORKDIR /
-RUN dpkg-deb --build bedops_2.4.20
+RUN dpkg-deb --build bedops_2.4.26
# deb file should now be located in / directory
diff --git a/packaging/deb/control b/packaging/deb/control
index f0f1742..b9dc7a0 100644
--- a/packaging/deb/control
+++ b/packaging/deb/control
@@ -1,5 +1,5 @@
Package: bedops
-Version: 2.4.20
+Version: 2.4.26
Section: base
Priority: optional
Architecture: amd64
diff --git a/packaging/os_x/BEDOPS.pkgproj b/packaging/os_x/BEDOPS.pkgproj
index b9bc39d..c6edc43 100755
--- a/packaging/os_x/BEDOPS.pkgproj
+++ b/packaging/os_x/BEDOPS.pkgproj
@@ -497,6 +497,22 @@
<key>GID</key>
<integer>0</integer>
<key>PATH</key>
+ <string>resources/bin/bam2bed_slurm</string>
+ <key>PATH_TYPE</key>
+ <integer>1</integer>
+ <key>PERMISSIONS</key>
+ <integer>493</integer>
+ <key>TYPE</key>
+ <integer>3</integer>
+ <key>UID</key>
+ <integer>0</integer>
+ </dict>
+ <dict>
+ <key>CHILDREN</key>
+ <array/>
+ <key>GID</key>
+ <integer>0</integer>
+ <key>PATH</key>
<string>resources/bin/bam2starch</string>
<key>PATH_TYPE</key>
<integer>1</integer>
@@ -545,6 +561,22 @@
<key>GID</key>
<integer>0</integer>
<key>PATH</key>
+ <string>resources/bin/bam2starch_slurm</string>
+ <key>PATH_TYPE</key>
+ <integer>1</integer>
+ <key>PERMISSIONS</key>
+ <integer>493</integer>
+ <key>TYPE</key>
+ <integer>3</integer>
+ <key>UID</key>
+ <integer>0</integer>
+ </dict>
+ <dict>
+ <key>CHILDREN</key>
+ <array/>
+ <key>GID</key>
+ <integer>0</integer>
+ <key>PATH</key>
<string>resources/bin/convert2bed</string>
<key>PATH_TYPE</key>
<integer>1</integer>
@@ -3536,6 +3568,54 @@
<key>UID</key>
<integer>0</integer>
</dict>
+ <dict>
+ <key>CHILDREN</key>
+ <array/>
+ <key>GID</key>
+ <integer>0</integer>
+ <key>PATH</key>
+ <string>resources/bin/update-sort-bed-migrate-candidates</string>
+ <key>PATH_TYPE</key>
+ <integer>1</integer>
+ <key>PERMISSIONS</key>
+ <integer>493</integer>
+ <key>TYPE</key>
+ <integer>3</integer>
+ <key>UID</key>
+ <integer>0</integer>
+ </dict>
+ <dict>
+ <key>CHILDREN</key>
+ <array/>
+ <key>GID</key>
+ <integer>0</integer>
+ <key>PATH</key>
+ <string>resources/bin/update-sort-bed-slurm</string>
+ <key>PATH_TYPE</key>
+ <integer>1</integer>
+ <key>PERMISSIONS</key>
+ <integer>493</integer>
+ <key>TYPE</key>
+ <integer>3</integer>
+ <key>UID</key>
+ <integer>0</integer>
+ </dict>
+ <dict>
+ <key>CHILDREN</key>
+ <array/>
+ <key>GID</key>
+ <integer>0</integer>
+ <key>PATH</key>
+ <string>resources/bin/update-sort-bed-starch-slurm</string>
+ <key>PATH_TYPE</key>
+ <integer>1</integer>
+ <key>PERMISSIONS</key>
+ <integer>493</integer>
+ <key>TYPE</key>
+ <integer>3</integer>
+ <key>UID</key>
+ <integer>0</integer>
+ </dict>
</array>
<key>GID</key>
<integer>0</integer>
@@ -4087,6 +4167,22 @@
<key>GID</key>
<integer>0</integer>
<key>PATH</key>
+ <string>resources/bin/starch-diff</string>
+ <key>PATH_TYPE</key>
+ <integer>1</integer>
+ <key>PERMISSIONS</key>
+ <integer>493</integer>
+ <key>TYPE</key>
+ <integer>3</integer>
+ <key>UID</key>
+ <integer>0</integer>
+ </dict>
+ <dict>
+ <key>CHILDREN</key>
+ <array/>
+ <key>GID</key>
+ <integer>0</integer>
+ <key>PATH</key>
<string>resources/bin/starchcat</string>
<key>PATH_TYPE</key>
<integer>1</integer>
@@ -4103,6 +4199,22 @@
<key>GID</key>
<integer>0</integer>
<key>PATH</key>
+ <string>resources/bin/starchstrip</string>
+ <key>PATH_TYPE</key>
+ <integer>1</integer>
+ <key>PERMISSIONS</key>
+ <integer>493</integer>
+ <key>TYPE</key>
+ <integer>3</integer>
+ <key>UID</key>
+ <integer>0</integer>
+ </dict>
+ <dict>
+ <key>CHILDREN</key>
+ <array/>
+ <key>GID</key>
+ <integer>0</integer>
+ <key>PATH</key>
<string>resources/bin/unstarch</string>
<key>PATH_TYPE</key>
<integer>1</integer>
@@ -4674,6 +4786,22 @@
<key>UID</key>
<integer>0</integer>
</dict>
+ <dict>
+ <key>CHILDREN</key>
+ <array/>
+ <key>GID</key>
+ <integer>0</integer>
+ <key>PATH</key>
+ <string>resources/bin/starchcluster_slurm</string>
+ <key>PATH_TYPE</key>
+ <integer>1</integer>
+ <key>PERMISSIONS</key>
+ <integer>493</integer>
+ <key>TYPE</key>
+ <integer>3</integer>
+ <key>UID</key>
+ <integer>0</integer>
+ </dict>
</array>
<key>GID</key>
<integer>0</integer>
@@ -5231,7 +5359,7 @@
<key>LANGUAGE</key>
<string>English</string>
<key>VALUE</key>
- <string>BEDOPS 2.4.20</string>
+ <string>BEDOPS 2.4.26</string>
</dict>
</array>
</dict>
@@ -5325,13 +5453,6 @@
<key>PATH_TYPE</key>
<integer>1</integer>
</dict>
- <key>CERTIFICATE</key>
- <dict>
- <key>NAME</key>
- <string>Developer ID Installer: Alex Reynolds (DLZRG65Q5B)</string>
- <key>PATH</key>
- <string>/Users/alexpreynolds/Library/Keychains/login.keychain</string>
- </dict>
<key>EXCLUDED_FILES</key>
<array>
<dict>
@@ -5501,7 +5622,7 @@
</dict>
</array>
<key>NAME</key>
- <string>BEDOPS 2.4.20</string>
+ <string>BEDOPS 2.4.26</string>
</dict>
</dict>
<key>TYPE</key>
diff --git a/packaging/rpm/Dockerfile b/packaging/rpm/Dockerfile
index 46c87e5..604e049 100644
--- a/packaging/rpm/Dockerfile
+++ b/packaging/rpm/Dockerfile
@@ -11,9 +11,9 @@ RUN yum -y install tar \
# copy the source context into the local image & build/install
# note: make sure .dockerignore is up to date
-RUN mkdir /bedops-2.4.20
-ADD . /bedops-2.4.20
-RUN tar zcf /bedops-2.4.20.tar.gz bedops-2.4.20
-RUN rm -rf /bedops-2.4.20
-RUN rpmbuild -ta bedops-2.4.20.tar.gz
-RUN rm /bedops-2.4.20.tar.gz
+RUN mkdir /bedops-2.4.26
+ADD . /bedops-2.4.26
+RUN tar zcf /bedops-2.4.26.tar.gz bedops-2.4.26
+RUN rm -rf /bedops-2.4.26
+RUN rpmbuild -ta bedops-2.4.26.tar.gz
+RUN rm /bedops-2.4.26.tar.gz
diff --git a/packaging/rpm/bedops.spec b/packaging/rpm/bedops.spec
index 664743b..1ec8295 100644
--- a/packaging/rpm/bedops.spec
+++ b/packaging/rpm/bedops.spec
@@ -1,5 +1,5 @@
Name: bedops
-Version: 2.4.20
+Version: 2.4.26
Release: 1
Summary: A suite of tools to address common questions raised in genomic studies.
Group: Applications/Productivity
diff --git a/system.mk/Makefile.darwin b/system.mk/Makefile.darwin
index dafd81a..14979fe 100644
--- a/system.mk/Makefile.darwin
+++ b/system.mk/Makefile.darwin
@@ -68,6 +68,9 @@ dbg_build_fat: dbg_sort_c_intel_fat dbg_bedops_c_intel_fat dbg_closestfeatures_c
#
clean: clean_debug clean_gprof all
rm -f ${BINDIR}/sort-bed
+ rm -f ${BINDIR}/update-sort-bed-slurm
+ rm -f ${BINDIR}/update-sort-bed-starch-slurm
+ rm -f ${BINDIR}/update-sort-bed-migrate-candidates
rm -f ${BINDIR}/bedops
rm -f ${BINDIR}/closest-features
rm -f ${BINDIR}/bedmap
@@ -75,7 +78,9 @@ clean: clean_debug clean_gprof all
rm -f ${BINDIR}/starch
rm -f ${BINDIR}/unstarch
rm -f ${BINDIR}/starchcat
+ rm -f ${BINDIR}/starchstrip
rm -f ${BINDIR}/starchcluster*
+ rm -f ${BINDIR}/starch-diff
rm -f ${BINDIR}/bam2bed
rm -f ${BINDIR}/bam2bed*
rm -f ${BINDIR}/gff2bed
@@ -113,6 +118,9 @@ clean: clean_debug clean_gprof all
clean_debug:
rm -f ${BINDIR}/debug.sort-bed
+ rm -f ${BINDIR}/update-sort-bed-slurm
+ rm -f ${BINDIR}/update-sort-bed-starch-slurm
+ rm -f ${BINDIR}/update-sort-bed-migrate-candidates
rm -f ${BINDIR}/debug.bedops
rm -f ${BINDIR}/debug.closest-features
rm -f ${BINDIR}/debug.bedmap
@@ -120,7 +128,9 @@ clean_debug:
rm -f ${BINDIR}/debug.starch
rm -f ${BINDIR}/debug.unstarch
rm -f ${BINDIR}/debug.starchcat
+ rm -f ${BINDIR}/debug.starchstrip
rm -f ${BINDIR}/starchcluster*
+ rm -f ${BINDIR}/starch-diff
rm -f ${BINDIR}/bam2bed
rm -f ${BINDIR}/bam2bed*
rm -f ${BINDIR}/gff2bed
@@ -146,6 +156,9 @@ clean_debug:
clean_gprof:
rm -f ${BINDIR}/gprof.sort-bed
+ rm -f ${BINDIR}/update-sort-bed-slurm
+ rm -f ${BINDIR}/update-sort-bed-starch-slurm
+ rm -f ${BINDIR}/update-sort-bed-migrate-candidates
rm -f ${BINDIR}/gprof.bedops
rm -f ${BINDIR}/gprof.closest-features
rm -f ${BINDIR}/gprof.bedmap
@@ -153,7 +166,9 @@ clean_gprof:
rm -f ${BINDIR}/gprof.starch
rm -f ${BINDIR}/gprof.unstarch
rm -f ${BINDIR}/gprof.starchcat
+ rm -f ${BINDIR}/gprof.starchstrip
rm -f ${BINDIR}/starchcluster*
+ rm -f ${BINDIR}/starch-diff
rm -f ${BINDIR}/bam2bed
rm -f ${BINDIR}/bam2bed*
rm -f ${BINDIR}/gff2bed
@@ -255,9 +270,12 @@ convert2bed_c_intel_fat:
starch_c_intel_fat:
cp ${APPDIR}/starch/src/starchcluster_sge.tcsh ${APPDIR}/starch/bin/starchcluster_sge
cp ${APPDIR}/starch/src/starchcluster_gnuParallel.tcsh ${APPDIR}/starch/bin/starchcluster_gnuParallel
+ cp ${APPDIR}/starch/src/starchcluster_slurm.tcsh ${APPDIR}/starch/bin/starchcluster_slurm
+ cp ${APPDIR}/starch/src/starch-diff.py ${APPDIR}/starch/bin/starch-diff
-test -e ${APPDIR}/starch/bin_i386/starch && test -e ${APPDIR}/starch/bin_x86_64/starch && { lipo -create ${APPDIR}/starch/bin_i386/starch ${APPDIR}/starch/bin_x86_64/starch -output ${APPDIR}/starch/bin/starch; }
-test -e ${APPDIR}/starch/bin_i386/unstarch && test -e ${APPDIR}/starch/bin_x86_64/unstarch && { lipo -create ${APPDIR}/starch/bin_i386/unstarch ${APPDIR}/starch/bin_x86_64/unstarch -output ${APPDIR}/starch/bin/unstarch; }
-test -e ${APPDIR}/starch/bin_i386/starchcat && test -e ${APPDIR}/starch/bin_x86_64/starchcat && { lipo -create ${APPDIR}/starch/bin_i386/starchcat ${APPDIR}/starch/bin_x86_64/starchcat -output ${APPDIR}/starch/bin/starchcat; }
+ -test -e ${APPDIR}/starch/bin_i386/starchstrip && test -e ${APPDIR}/starch/bin_x86_64/starchstrip && { lipo -create ${APPDIR}/starch/bin_i386/starchstrip ${APPDIR}/starch/bin_x86_64/starchstrip -output ${APPDIR}/starch/bin/starchstrip; }
@@ -285,6 +303,9 @@ dbg_convert2bed_c_intel_fat:
dbg_starch_c_intel_fat:
cp ${APPDIR}/starch/src/starchcluster_sge.tcsh ${APPDIR}/starch/bin/starchcluster_sge
cp ${APPDIR}/starch/src/starchcluster_gnuParallel.tcsh ${APPDIR}/starch/bin/starchcluster_gnuParallel
+ cp ${APPDIR}/starch/src/starchcluster_slurm.tcsh ${APPDIR}/starch/bin/starchcluster_slurm
+ cp ${APPDIR}/starch/src/starch-diff.py ${APPDIR}/starch/bin/starch-diff
-test -e ${APPDIR}/starch/bin_i386/debug.starch && test -e ${APPDIR}/starch/bin_x86_64/debug.starch && { lipo -create ${APPDIR}/starch/bin_i386/debug.starch ${APPDIR}/starch/bin_x86_64/debug.starch -output ${APPDIR}/starch/bin/debug.starch; }
-test -e ${APPDIR}/starch/bin_i386/debug.unstarch && test -e ${APPDIR}/starch/bin_x86_64/debug.unstarch && { lipo -create ${APPDIR}/starch/bin_i386/debug.unstarch ${APPDIR}/starch/bin_x86_64/debug.unstarch -output ${APPDIR}/starch/bin/debug.unstarch; }
-test -e ${APPDIR}/starch/bin_i386/debug.starchcat && test -e ${APPDIR}/starch/bin_x86_64/debug.starchcat && { lipo -create ${APPDIR}/starch/bin_i386/debug.starchcat ${APPDIR}/starch/bin_x86_64/debug.starchcat -output ${APPDIR}/starch/bin/debug.starchcat; }
+ -test -e ${APPDIR}/starch/bin_i386/debug.starchstrip && test -e ${APPDIR}/starch/bin_x86_64/debug.starchstrip && { lipo -create ${APPDIR}/starch/bin_i386/debug.starchstrip ${APPDIR}/starch/bin_x86_64/debug.starchstrip -output ${APPDIR}/starch/bin/debug.starchstrip; }
diff --git a/system.mk/Makefile.linux b/system.mk/Makefile.linux
index 1e18d8b..236ae0a 100644
--- a/system.mk/Makefile.linux
+++ b/system.mk/Makefile.linux
@@ -40,6 +40,9 @@ build_gprof: $(SUBDIRS)
clean: clean_debug clean_gprof $(SUBDIRS)
rm -f ${BINDIR}/sort-bed
+ rm -f ${BINDIR}/update-sort-bed-slurm
+ rm -f ${BINDIR}/update-sort-bed-starch-slurm
+ rm -f ${BINDIR}/update-sort-bed-migrate-candidates
rm -f ${BINDIR}/bedops
rm -f ${BINDIR}/closest-features
rm -f ${BINDIR}/bedmap
@@ -47,7 +50,9 @@ clean: clean_debug clean_gprof $(SUBDIRS)
rm -f ${BINDIR}/starch
rm -f ${BINDIR}/unstarch
rm -f ${BINDIR}/starchcat
+ rm -f ${BINDIR}/starchstrip
rm -f ${BINDIR}/starchcluster*
+ rm -f ${BINDIR}/starch-diff
rm -f ${BINDIR}/bam2bed
rm -f ${BINDIR}/bam2bed*
rm -f ${BINDIR}/gff2bed
@@ -82,6 +87,9 @@ clean: clean_debug clean_gprof $(SUBDIRS)
clean_debug:
rm -f ${BINDIR}/debug.sort-bed
+ rm -f ${BINDIR}/update-sort-bed-slurm
+ rm -f ${BINDIR}/update-sort-bed-starch-slurm
+ rm -f ${BINDIR}/update-sort-bed-migrate-candidates
rm -f ${BINDIR}/debug.bedops
rm -f ${BINDIR}/debug.closest-features
rm -f ${BINDIR}/debug.bedmap
@@ -89,7 +97,9 @@ clean_debug:
rm -f ${BINDIR}/debug.starch
rm -f ${BINDIR}/debug.unstarch
rm -f ${BINDIR}/debug.starchcat
+ rm -f ${BINDIR}/debug.starchstrip
rm -f ${BINDIR}/starchcluster*
+ rm -f ${BINDIR}/starch-diff
rm -f ${BINDIR}/bam2bed
rm -f ${BINDIR}/bam2bed*
rm -f ${BINDIR}/gff2bed
@@ -115,6 +125,9 @@ clean_debug:
clean_gprof:
rm -f ${BINDIR}/gprof.sort-bed
+ rm -f ${BINDIR}/update-sort-bed-slurm
+ rm -f ${BINDIR}/update-sort-bed-starch-slurm
+ rm -f ${BINDIR}/update-sort-bed-migrate-candidates
rm -f ${BINDIR}/gprof.bedops
rm -f ${BINDIR}/gprof.closest-features
rm -f ${BINDIR}/gprof.bedmap
@@ -122,7 +135,9 @@ clean_gprof:
rm -f ${BINDIR}/gprof.starch
rm -f ${BINDIR}/gprof.unstarch
rm -f ${BINDIR}/gprof.starchcat
+ rm -f ${BINDIR}/gprof.starchstrip
rm -f ${BINDIR}/starchcluster*
+ rm -f ${BINDIR}/starch-diff
rm -f ${BINDIR}/bam2bed
rm -f ${BINDIR}/bam2bed*
rm -f ${BINDIR}/gff2bed
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/bedops.git
More information about the debian-med-commit
mailing list